From 988a573f7fc1932aa4647c99dec1d33a93a44688 Mon Sep 17 00:00:00 2001 From: gordon chung Date: Mon, 17 Aug 2015 16:20:49 -0400 Subject: [PATCH] make telemetry sample payloads dictionaries this patch makes the payload we generate via polling agents and api a dictionary. this gives us more flexibility to include metadata on how the data was generated that we don't necessary want to include in sample metadata. Change-Id: I1350c78c52fad8111241b3f77698fef6c4aa77a9 Closes-Bug: #1484937 --- ceilometer/agent/base.py | 22 +++----- ceilometer/api/controllers/v2/meters.py | 3 +- ceilometer/telemetry/notifications.py | 4 +- .../api/v2/test_post_samples_scenarios.py | 5 +- ceilometer/tests/unit/agent/test_manager.py | 2 +- .../unit/telemetry/test_notifications.py | 55 +++++++++---------- 6 files changed, 42 insertions(+), 49 deletions(-) diff --git a/ceilometer/agent/base.py b/ceilometer/agent/base.py index 3b507dc6..d3e70205 100644 --- a/ceilometer/agent/base.py +++ b/ceilometer/agent/base.py @@ -117,10 +117,7 @@ class PollingTask(object): resource_factory = lambda: Resources(agent_manager) self.resources = collections.defaultdict(resource_factory) - if cfg.CONF.batch_polled_samples: - self._handle_sample = self._assemble_samples - else: - self._handle_sample = self._send_notification + self._batch = cfg.CONF.batch_polled_samples self._telemetry_secret = cfg.CONF.publisher.telemetry_secret def add(self, pollster, source): @@ -179,10 +176,11 @@ class PollingTask(object): publisher_utils.meter_message_from_counter( sample, self._telemetry_secret )) - self._handle_sample([sample_dict], sample_batch) + if self._batch: + sample_batch.append(sample_dict) + else: + self._send_notification([sample_dict]) - # sample_batch will contain samples if - # cfg.CONF.batch_polled_samples is True if sample_batch: self._send_notification(sample_batch) @@ -198,15 +196,11 @@ class PollingTask(object): % ({'name': pollster.name, 'error': err}), exc_info=True) - @staticmethod - def _assemble_samples(samples, batch): - batch.extend(samples) - - def _send_notification(self, samples, batch=None): + def _send_notification(self, samples): self.manager.notifier.info( self.manager.context.to_dict(), - 'telemetry.api', - samples + 'telemetry.polling', + {'samples': samples} ) diff --git a/ceilometer/api/controllers/v2/meters.py b/ceilometer/api/controllers/v2/meters.py index ff74240d..138ce15a 100644 --- a/ceilometer/api/controllers/v2/meters.py +++ b/ceilometer/api/controllers/v2/meters.py @@ -367,7 +367,8 @@ class MeterController(rest.RestController): tenant=def_project_id, is_admin=True) notifier = pecan.request.notifier - notifier.info(ctxt.to_dict(), 'telemetry.api', published_samples) + notifier.info(ctxt.to_dict(), 'telemetry.api', + {'samples': published_samples}) return samples diff --git a/ceilometer/telemetry/notifications.py b/ceilometer/telemetry/notifications.py index c9a98300..4c47c079 100644 --- a/ceilometer/telemetry/notifications.py +++ b/ceilometer/telemetry/notifications.py @@ -45,10 +45,10 @@ class TelemetryBase(plugin_base.NotificationBase): class TelemetryApiPost(TelemetryBase): """Handle sample from notification bus, which is posted via API.""" - event_types = ['telemetry.api'] + event_types = ['telemetry.api', 'telemetry.polling'] def process_notification(self, message): - samples = message['payload'] + samples = message['payload']['samples'] for sample_dict in samples: yield sample.Sample( name=sample_dict['counter_name'], diff --git a/ceilometer/tests/functional/api/v2/test_post_samples_scenarios.py b/ceilometer/tests/functional/api/v2/test_post_samples_scenarios.py index cc52ff88..0ec78c89 100644 --- a/ceilometer/tests/functional/api/v2/test_post_samples_scenarios.py +++ b/ceilometer/tests/functional/api/v2/test_post_samples_scenarios.py @@ -29,9 +29,10 @@ from ceilometer.tests.functional.api import v2 class TestPostSamples(v2.FunctionalTest, tests_db.MixinTestsWithBackendScenarios): def fake_notifier_sample(self, ctxt, event_type, payload): - for m in payload: + samples = payload['samples'] + for m in samples: del m['message_signature'] - self.published.append(payload) + self.published.append(samples) def setUp(self): self.published = [] diff --git a/ceilometer/tests/unit/agent/test_manager.py b/ceilometer/tests/unit/agent/test_manager.py index e2d37bf0..3f61f13d 100644 --- a/ceilometer/tests/unit/agent/test_manager.py +++ b/ceilometer/tests/unit/agent/test_manager.py @@ -201,7 +201,7 @@ class TestRunTasks(agentbase.BaseAgentManagerTestCase): return pipeline_cfg_file def fake_notifier_sample(self, ctxt, event_type, payload): - for m in payload: + for m in payload['samples']: del m['message_signature'] self.notified_samples.append(m) diff --git a/ceilometer/tests/unit/telemetry/test_notifications.py b/ceilometer/tests/unit/telemetry/test_notifications.py index 2584c0f6..e6a1e3b8 100644 --- a/ceilometer/tests/unit/telemetry/test_notifications.py +++ b/ceilometer/tests/unit/telemetry/test_notifications.py @@ -21,34 +21,31 @@ NOTIFICATION = { 'timestamp': u'2015-06-1909: 19: 35.786893', u'_context_auth_token': None, u'_context_read_only': False, - 'payload': [{ - u'counter_name': u'instance100', - u'user_id': u'e1d870e51c7340cb9d555b15cbfcaec2', - u'resource_id': u'instance', - u'timestamp': u'2015-06-19T09: 19: 35.785330', - u'message_signature': u'fake_signature1', - u'resource_metadata': {u'foo': u'bar'}, - u'source': u'30be1fc9a03c4e94ab05c403a8a377f2: openstack', - u'counter_unit': u'instance', - u'counter_volume': 1.0, - u'project_id': u'30be1fc9a03c4e94ab05c403a8a377f2', - u'message_id': u'4d865c6e-1664-11e5-9d41-0819a6cff905', - u'counter_type': u'gauge' - }, - { - u'counter_name': u'instance100', - u'user_id': u'e1d870e51c7340cb9d555b15cbfcaec2', - u'resource_id': u'instance', - u'timestamp': u'2015-06-19T09: 19: 35.785330', - u'message_signature': u'fake_signature12', - u'resource_metadata': {u'foo': u'bar'}, - u'source': u'30be1fc9a03c4e94ab05c403a8a377f2: openstack', - u'counter_unit': u'instance', - u'counter_volume': 1.0, - u'project_id': u'30be1fc9a03c4e94ab05c403a8a377f2', - u'message_id': u'4d866da8-1664-11e5-9d41-0819a6cff905', - u'counter_type': u'gauge' - }], + 'payload': {'samples': + [{'counter_name': u'instance100', + u'user_id': u'e1d870e51c7340cb9d555b15cbfcaec2', + u'resource_id': u'instance', + u'timestamp': u'2015-06-19T09: 19: 35.785330', + u'message_signature': u'fake_signature1', + u'resource_metadata': {u'foo': u'bar'}, + u'source': u'30be1fc9a03c4e94ab05c403a8a377f2: openstack', + u'counter_unit': u'instance', + u'counter_volume': 1.0, + u'project_id': u'30be1fc9a03c4e94ab05c403a8a377f2', + u'message_id': u'4d865c6e-1664-11e5-9d41-0819a6cff905', + u'counter_type': u'gauge'}, + {u'counter_name': u'instance100', + u'user_id': u'e1d870e51c7340cb9d555b15cbfcaec2', + u'resource_id': u'instance', + u'timestamp': u'2015-06-19T09: 19: 35.785330', + u'message_signature': u'fake_signature12', + u'resource_metadata': {u'foo': u'bar'}, + u'source': u'30be1fc9a03c4e94ab05c403a8a377f2: openstack', + u'counter_unit': u'instance', + u'counter_volume': 1.0, + u'project_id': u'30be1fc9a03c4e94ab05c403a8a377f2', + u'message_id': u'4d866da8-1664-11e5-9d41-0819a6cff905', + u'counter_type': u'gauge'}]}, u'_context_resource_uuid': None, u'_context_user_identity': u'fake_user_identity---', u'_context_show_deleted': False, @@ -69,7 +66,7 @@ class TelemetryApiPostTestCase(base.BaseTestCase): sample_creation = notifications.TelemetryApiPost(None) samples = list(sample_creation.process_notification(NOTIFICATION)) self.assertEqual(2, len(samples)) - payload = NOTIFICATION["payload"] + payload = NOTIFICATION["payload"]['samples'] for index, sample in enumerate(samples): self.assertEqual(payload[index]["user_id"], sample.user_id) self.assertEqual(payload[index]["counter_name"], sample.name)