Add timestamps and metadata into creating samples

add timestamps with certain interval and resource_metadata into creating
samples in sample_generator

Change-Id: Id24487eb1cd86da065e2d21584b7dda8ebe112cb

rally-jobs/rally.yaml

@@ -480,6 +480,16 @@
           counter_volume: 1.0
           resources_per_tenant: 3
           samples_per_resource: 3
+          timestamp_interval: 60
+          metadata_list:
+            -  status: "active"
+               name: "fake_resource"
+               deleted: "False"
+               created_at: "2015-09-04T12:34:19.000000"
+            -  status: "not_active"
+               name: "fake_resource_1"
+               deleted: "False"
+               created_at: "2015-09-10T06:55:12.000000"
       sla:
         failure_rate:
           max: 0

rally/plugins/openstack/context/ceilometer/samples.py

@@ -54,6 +54,30 @@ class CeilometerSampleGenerator(context.Context):
                 "type": "integer",
                 "minimum": 1
             },
+            "timestamp_interval": {
+                "type": "integer",
+                "minimum": 1
+            },
+            "metadata_list": {
+                "type": "array",
+                "items": {
+                    "type": "object",
+                    "properties": {
+                        "status": {
+                            "type": "string"
+                        },
+                        "name": {
+                            "type": "string"
+                        },
+                        "deleted": {
+                            "type": "string"
+                        },
+                        "created_at": {
+                            "type": "string"
+                        }
+                    }
+                }
+            }
         },
         "required": ["counter_name", "counter_type", "counter_unit",
                      "counter_volume"],
@@ -62,7 +86,8 @@ class CeilometerSampleGenerator(context.Context):
 
     DEFAULT_CONFIG = {
         "resources_per_tenant": 5,
-        "samples_per_resource": 5
+        "samples_per_resource": 5,
+        "timestamp_interval": 60
     }
 
     @logging.log_task_wrapper(LOG.info, _("Enter context: `Ceilometer`"))
@@ -71,7 +96,7 @@ class CeilometerSampleGenerator(context.Context):
             "counter_name": self.config["counter_name"],
             "counter_type": self.config["counter_type"],
             "counter_unit": self.config["counter_unit"],
-            "counter_volume": self.config["counter_volume"]
+            "counter_volume": self.config["counter_volume"],
         }
         for user, tenant_id in rutils.iterate_per_tenants(
                 self.context["users"]):
@@ -82,7 +107,10 @@ class CeilometerSampleGenerator(context.Context):
             )
             for i in moves.xrange(self.config["resources_per_tenant"]):
                 samples_to_create = scenario._make_samples(
-                    count=self.config["samples_per_resource"], **new_sample)
+                    count=self.config["samples_per_resource"],
+                    interval=self.config["timestamp_interval"],
+                    metadata_list=self.config.get("metadata_list"),
+                    **new_sample)
                 samples = scenario._create_samples(samples_to_create)
                 for sample in samples:
                     self.context["tenants"][tenant_id]["samples"].append(

rally/plugins/openstack/scenarios/ceilometer/utils.py

@@ -11,6 +11,8 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
+import datetime
+
 import six
 
 from rally.plugins.openstack import scenario
@@ -21,13 +23,15 @@ from rally.task import utils as bench_utils
 class CeilometerScenario(scenario.OpenStackScenario):
     """Base class for Ceilometer scenarios with basic atomic actions."""
 
-    def _make_samples(self, count=1, counter_name="cpu_util",
+    def _make_samples(self, count=1, interval=0, counter_name="cpu_util",
                       counter_type="gauge", counter_unit="%", counter_volume=1,
                       project_id=None, user_id=None, source=None,
-                      timestamp=None, resource_metadata=None):
+                      timestamp=None, metadata_list=None):
         """Prepare and return a list of samples.
 
         :param count: specifies number of samples in array
+        :param interval: specifies interval between timestamps of near-by
+        samples
         :param counter_name: specifies name of the counter
         :param counter_type: specifies type of the counter
         :param counter_unit: specifies unit of the counter
@@ -36,9 +40,10 @@ class CeilometerScenario(scenario.OpenStackScenario):
         :param user_id: specifies user id for samples
         :param source: specifies source for samples
         :param timestamp: specifies timestamp for samples
-        :param resource_metadata: specifies resource metadata
+        :param metadata_list: specifies list of resource metadata
         :returns: list of samples used to create samples
         """
+        samples = []
         sample = {
             "counter_name": counter_name,
             "counter_type": counter_type,
@@ -51,13 +56,25 @@ class CeilometerScenario(scenario.OpenStackScenario):
             "user_id": user_id,
             "source": source,
             "timestamp": timestamp,
-            "resource_metadata": resource_metadata,
         }
         for k, v in six.iteritems(opt_fields):
             if v:
                 sample.update({k: v})
+        now = timestamp or datetime.datetime.utcnow()
+        len_meta = len(metadata_list) if metadata_list else 0
+        for i in six.moves.xrange(count):
+            sample_item = dict(sample)
+            sample_item["timestamp"] = (
+                now - datetime.timedelta(seconds=(interval * i))
+            ).isoformat()
+            if metadata_list:
+                # NOTE(idegtiarov): Adding more then one template of metadata
+                # required it's proportional distribution among whole samples.
+                sample_item["resource_metadata"] = metadata_list[
+                    i * len_meta // count
+                ]
+            samples.append(sample_item)
 
-        samples = [sample] * count
         return samples
 
     def _get_alarm_dict(self, **kwargs):

samples/tasks/scenarios/ceilometer/list-samples.json

@@ -17,7 +17,16 @@
                     "counter_unit": "instance",
                     "counter_volume": 1.0,
                     "resources_per_tenant": 3,
-                    "samples_per_resource": 3
+                    "samples_per_resource": 3,
+                    "timestamp_interval": 60,
+                    "metadata_list": [
+                        {"status": "active", "name": "fake_resource",
+                         "deleted": "False",
+                         "created_at": "2015-09-04T12:34:19.000000"},
+                        {"status": "not_active", "name": "fake_resource_1",
+                         "deleted": "False",
+                         "created_at": "2015-09-10T06:55:12.000000"}
+                    ]
                 }
             }
         }

samples/tasks/scenarios/ceilometer/list-samples.yaml

@@ -16,3 +16,13 @@
            counter_volume: 1.0
            resources_per_tenant: 3
            samples_per_resource: 3
+           timestamp_interval: 60
+           metadata_list:
+             -  status: "active"
+                name: "fake_resource"
+                deleted: "False"
+                created_at: "2015-09-04T12:34:19.000000"
+             -  status: "not_active"
+                name: "fake_resource_1"
+                deleted: "False"
+                created_at: "2015-09-10T06:55:12.000000"

test-requirements.txt

@@ -6,6 +6,7 @@ hacking<0.10,>=0.9.2
 coverage>=3.6
 ddt>=0.7.0
 mock>=1.2
+python-dateutil>=2.4.2
 testrepository>=0.0.18
 testtools>=1.4.0
 

tests/unit/plugins/openstack/context/ceilometer/test_samples.py

@@ -12,12 +12,13 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-
 import copy
 
 import mock
+import six
 
 from rally.plugins.openstack.context.ceilometer import samples
+from rally.plugins.openstack.scenarios.ceilometer import utils as ceilo_utils
 from tests.unit import test
 
 CTX = "rally.plugins.openstack.context.ceilometer"
@@ -53,7 +54,16 @@ class CeilometerSampleGeneratorTestCase(test.TestCase):
                     "counter_unit": "fake-counter-unit",
                     "counter_volume": 100,
                     "resources_per_tenant": resources_per_tenant,
-                    "samples_per_resource": samples_per_resource
+                    "samples_per_resource": samples_per_resource,
+                    "timestamp_interval": 60,
+                    "metadata_list": [
+                        {"status": "active", "name": "fake_resource",
+                         "deleted": "False",
+                         "created_at": "2015-09-04T12:34:19.000000"},
+                        {"status": "not_active", "name": "fake_resource_1",
+                         "deleted": "False",
+                         "created_at": "2015-09-10T06:55:12.000000"}
+                    ]
                 }
             },
             "admin": {
@@ -74,16 +84,23 @@ class CeilometerSampleGeneratorTestCase(test.TestCase):
                 "counter_unit": "instance",
                 "counter_volume": 1.0,
                 "resources_per_tenant": 5,
-                "samples_per_resource": 5
+                "samples_per_resource": 5,
+                "timestamp_intervals": 60,
+                "metadata_list": [
+                    {"status": "active", "name": "fake_resource",
+                     "deleted": "False",
+                     "created_at": "2015-09-04T12:34:19.000000"},
+                    {"status": "not_active", "name": "fake_resource_1",
+                     "deleted": "False",
+                     "created_at": "2015-09-10T06:55:12.000000"}
+                ]
             }
         }
 
         inst = samples.CeilometerSampleGenerator(context)
         self.assertEqual(inst.config, context["config"]["ceilometer"])
 
-    @mock.patch("%s.samples.ceilo_utils.CeilometerScenario._create_samples"
+    def test_setup(self):
-                % CTX)
-    def test_setup(self, mock_ceilometer_scenario__create_samples):
         tenants_count = 2
         users_per_tenant = 2
         resources_per_tenant = 2
@@ -92,32 +109,47 @@ class CeilometerSampleGeneratorTestCase(test.TestCase):
         tenants, real_context = self._gen_context(
             tenants_count, users_per_tenant,
             resources_per_tenant, samples_per_resource)
-
+        scenario = ceilo_utils.CeilometerScenario(real_context)
         sample = {
             "counter_name": "fake-counter-name",
             "counter_type": "fake-counter-type",
             "counter_unit": "fake-counter-unit",
             "counter_volume": 100,
             "resource_id": "fake-resource-id",
+            "metadata_list": [
+                {"status": "active", "name": "fake_resource",
+                 "deleted": "False",
+                 "created_at": "2015-09-04T12:34:19.000000"},
+                {"status": "not_active", "name": "fake_resource_1",
+                 "deleted": "False",
+                 "created_at": "2015-09-10T06:55:12.000000"}
+            ]
         }
-
+        scenario.generate_random_name = mock.Mock(
+            return_value="fake_resource-id")
+        kwargs = copy.deepcopy(sample)
+        kwargs.pop("resource_id")
+        samples_to_create = scenario._make_samples(count=samples_per_resource,
+                                                   interval=60, **kwargs)
         new_context = copy.deepcopy(real_context)
         for id_ in tenants.keys():
             new_context["tenants"][id_].setdefault("samples", [])
             new_context["tenants"][id_].setdefault("resources", [])
-            for i in range(resources_per_tenant):
+            for i in six.moves.xrange(resources_per_tenant):
-                for j in range(samples_per_resource):
+                for sample in samples_to_create:
                     new_context["tenants"][id_]["samples"].append(sample)
                 new_context["tenants"][id_]["resources"].append(
                     sample["resource_id"])
-
+        with mock.patch("%s.samples.ceilo_utils.CeilometerScenario"
-        mock_ceilometer_scenario__create_samples.return_value = [
+                        "._create_samples" % CTX) as mock_create_samples:
-            mock.MagicMock(to_dict=lambda: sample, **sample)
+            mock_create_samples.return_value = []
-            for i in range(samples_per_resource)]
+            for i, sample in enumerate(samples_to_create):
-
+                sample_object = mock.MagicMock(resource_id="fake_resource-id")
-        ceilometer_ctx = samples.CeilometerSampleGenerator(real_context)
+                sample_object.to_dict.return_value = sample
-        ceilometer_ctx.setup()
+                mock_create_samples.return_value.append(sample_object)
-        self.assertEqual(new_context, ceilometer_ctx.context)
+            ceilometer_ctx = samples.CeilometerSampleGenerator(real_context)
+            ceilometer_ctx.setup()
+            self.assertEqual(new_context, ceilometer_ctx.context)
 
     def test_cleanup(self):
         tenants, context = self._gen_context(2, 5, 3, 3)

tests/unit/plugins/openstack/scenarios/ceilometer/test_utils.py

@@ -13,7 +13,9 @@
 #    under the License.
 
 import copy
+import datetime
 
+from dateutil import parser
 import mock
 
 from rally.plugins.openstack.scenarios.ceilometer import utils
@@ -30,15 +32,19 @@ class CeilometerScenarioTestCase(test.ScenarioTestCase):
     def test__make_samples(self):
         self.scenario.generate_random_name = mock.Mock(
             return_value="fake_resource")
-        result = self.scenario._make_samples(project_id="fake_project_id")
+        test_timestamp = datetime.datetime(2015, 10, 20, 14, 18, 40)
-        expected = [{"counter_name": "cpu_util",
+        result = self.scenario._make_samples(count=2, interval=60,
-                     "counter_type": "gauge",
+                                             timestamp=test_timestamp)
-                     "counter_unit": "%",
+        expected = {"counter_name": "cpu_util",
-                     "counter_volume": 1,
+                    "counter_type": "gauge",
-                     "resource_id": "fake_resource",
+                    "counter_unit": "%",
-                     "project_id": "fake_project_id",
+                    "counter_volume": 1,
-                     }]
+                    "resource_id": "fake_resource",
-        self.assertEqual(expected, result)
+                    "timestamp": test_timestamp.isoformat()}
+        self.assertEqual(expected, result[0])
+        samples_int = (parser.parse(result[0]["timestamp"]) -
+                       parser.parse(result[1]["timestamp"])).seconds
+        self.assertEqual(60, samples_int)
 
     def test__list_alarms_by_id(self):
         self.assertEqual(self.clients("ceilometer").alarms.get.return_value,