From a1f02758014ff26f4604b8e680c489213f4fdc8c Mon Sep 17 00:00:00 2001 From: Mehdi Abaakouk Date: Tue, 22 Nov 2016 13:16:02 +0100 Subject: [PATCH] gnocchi: use batch create_metrics=True Change-Id: I2de9e886a26b60751cc5816f6257b28218d14e69 --- ceilometer/dispatcher/gnocchi.py | 41 +++++-------------- .../tests/unit/dispatcher/test_gnocchi.py | 39 ++++++------------ 2 files changed, 23 insertions(+), 57 deletions(-) diff --git a/ceilometer/dispatcher/gnocchi.py b/ceilometer/dispatcher/gnocchi.py index 89849da36c..2bad6cfefd 100644 --- a/ceilometer/dispatcher/gnocchi.py +++ b/ceilometer/dispatcher/gnocchi.py @@ -17,7 +17,6 @@ import hashlib import itertools import operator import pkg_resources -import re import threading import uuid @@ -385,56 +384,36 @@ class GnocchiDispatcher(dispatcher.MeterDispatcherBase, except Exception as e: LOG.error(six.text_type(e), exc_info=True) - RE_UNKNOW_METRICS = re.compile("Unknown metrics: (.*) \(HTTP 400\)") - RE_UNKNOW_METRICS_LIST = re.compile("([^/ ,]*)/([^,]*)") - def batch_measures(self, measures, resource_infos, stats): # NOTE(sileht): We don't care about error here, we want # resources metadata always been updated try: - self._gnocchi.metric.batch_resources_metrics_measures(measures) + self._gnocchi.metric.batch_resources_metrics_measures( + measures, create_metrics=True) except gnocchi_exc.BadRequest as e: - m = self.RE_UNKNOW_METRICS.match(six.text_type(e)) - if m is None: + if not isinstance(e.message, dict): + raise + if e.message.get('cause') != 'Unknown resources': raise - # NOTE(sileht): Create all missing resources and metrics - metric_list = self.RE_UNKNOW_METRICS_LIST.findall(m.group(1)) - gnocchi_ids_freshly_handled = set() - for gnocchi_id, metric_name in metric_list: - if gnocchi_id in gnocchi_ids_freshly_handled: - continue + for gnocchi_id in e.message['detail']: resource = resource_infos[gnocchi_id]['resource'] resource_type = resource_infos[gnocchi_id]['resource_type'] try: self._if_not_cached("create", resource_type, resource, self._create_resource) except gnocchi_exc.ResourceAlreadyExists: - metric = {'resource_id': resource['id'], - 'name': metric_name} - metric.update(resource["metrics"][metric_name]) - try: - self._gnocchi.metric.create(metric) - except gnocchi_exc.NamedMetricAlreadyExists: - # NOTE(sileht): metric created in the meantime - pass - except gnocchi_exc.ClientException as e: - LOG.error(six.text_type(e)) - # We cannot post measures for this metric - del measures[gnocchi_id][metric_name] - if not measures[gnocchi_id]: - del measures[gnocchi_id] + # NOTE(sileht): resource created in the meantime + pass except gnocchi_exc.ClientException as e: LOG.error(six.text_type(e)) # We cannot post measures for this resource del measures[gnocchi_id] - gnocchi_ids_freshly_handled.add(gnocchi_id) - else: - gnocchi_ids_freshly_handled.add(gnocchi_id) # NOTE(sileht): we have created missing resources/metrics, # now retry to post measures - self._gnocchi.metric.batch_resources_metrics_measures(measures) + self._gnocchi.metric.batch_resources_metrics_measures( + measures, create_metrics=True) # FIXME(sileht): take care of measures removed in stats LOG.debug("%(measures)d measures posted against %(metrics)d " diff --git a/ceilometer/tests/unit/dispatcher/test_gnocchi.py b/ceilometer/tests/unit/dispatcher/test_gnocchi.py index 81e5aaa429..33451ada58 100644 --- a/ceilometer/tests/unit/dispatcher/test_gnocchi.py +++ b/ceilometer/tests/unit/dispatcher/test_gnocchi.py @@ -452,10 +452,9 @@ class DispatcherWorkflowTest(base.BaseTestCase, ] default_workflow = dict(resource_exists=True, - metric_exists=True, post_measure_fail=False, create_resource_fail=False, - create_metric_fail=False, + create_resource_race=False, update_resource_fail=False, retry_post_measures_fail=False) workflow_scenarios = [ @@ -463,10 +462,9 @@ class DispatcherWorkflowTest(base.BaseTestCase, ('new_resource', dict(resource_exists=False)), ('new_resource_fail', dict(resource_exists=False, create_resource_fail=True)), + ('new_resource_race', dict(resource_exists=False, + create_resource_race=True)), ('resource_update_fail', dict(update_resource_fail=True)), - ('new_metric', dict(metric_exists=False)), - ('new_metric_fail', dict(metric_exists=False, - create_metric_fail=True)), ('retry_fail', dict(resource_exists=False, retry_post_measures_fail=True)), ('measure_fail', dict(post_measure_fail=True)), @@ -572,7 +570,8 @@ class DispatcherWorkflowTest(base.BaseTestCase, expected_calls = [ mock.call.capabilities.list(), mock.call.metric.batch_resources_metrics_measures( - {gnocchi_id: {metric_name: self.measures_attributes}}) + {gnocchi_id: {metric_name: self.measures_attributes}}, + create_metrics=True) ] expected_debug = [ mock.call('gnocchi project found: %s', @@ -583,11 +582,11 @@ class DispatcherWorkflowTest(base.BaseTestCase, batch_side_effect = [] if self.post_measure_fail: batch_side_effect += [Exception('boom!')] - elif not self.resource_exists or not self.metric_exists: + elif not self.resource_exists: batch_side_effect += [ gnocchi_exc.BadRequest( - 400, "Unknown metrics: %s/%s" % (gnocchi_id, - metric_name))] + 400, {"cause": "Unknown resources", + 'detail': [gnocchi_id]})] attributes = self.postable_attributes.copy() attributes.update(self.patchable_attributes) attributes['id'] = self.sample['resource_id'] @@ -605,30 +604,18 @@ class DispatcherWorkflowTest(base.BaseTestCase, if self.create_resource_fail: fakeclient.resource.create.side_effect = [Exception('boom!')] - elif self.resource_exists: + elif self.create_resource_race: fakeclient.resource.create.side_effect = [ gnocchi_exc.ResourceAlreadyExists(409)] - - expected_calls.append(mock.call.metric.create({ - 'name': self.sample['counter_name'], - 'unit': self.sample['counter_unit'], - 'resource_id': resource_id})) - if self.create_metric_fail: - fakeclient.metric.create.side_effect = [Exception('boom!')] - elif self.metric_exists: - fakeclient.metric.create.side_effect = [ - gnocchi_exc.NamedMetricAlreadyExists(409)] - else: - fakeclient.metric.create.side_effect = [None] - else: # not resource_exists expected_debug.append(mock.call( 'Resource %s created', self.sample['resource_id'])) - if not self.create_resource_fail and not self.create_metric_fail: + if not self.create_resource_fail: expected_calls.append( mock.call.metric.batch_resources_metrics_measures( - {gnocchi_id: {metric_name: self.measures_attributes}}) + {gnocchi_id: {metric_name: self.measures_attributes}}, + create_metrics=True) ) if self.retry_post_measures_fail: @@ -664,7 +651,7 @@ class DispatcherWorkflowTest(base.BaseTestCase, self.dispatcher.record_metering_data([self.sample]) # Check that the last log message is the expected one - if (self.post_measure_fail or self.create_metric_fail + if (self.post_measure_fail or self.create_resource_fail or self.retry_post_measures_fail or (self.update_resource_fail and self.patchable_attributes)):