openstack/ceilometer
Code Issues Proposed changes

gnocchi: use batch create_metrics=True

Browse Source 
Change-Id: I2de9e886a26b60751cc5816f6257b28218d14e69
This commit is contained in:
Mehdi Abaakouk 2016-11-22 13:16:02 +01:00
parent 45aa43966f
commit a1f0275801
2 changed files with 23 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
ceilometer/dispatcher/gnocchi.py
View File

@ -17,7 +17,6 @@ import hashlib
import itertools import itertools
import operator import operator
import pkg_resources import pkg_resources
import re
import threading import threading
import uuid import uuid
@ -385,56 +384,36 @@ class GnocchiDispatcher(dispatcher.MeterDispatcherBase,
except Exception as e: except Exception as e:
LOG.error(six.text_type(e), exc_info=True) LOG.error(six.text_type(e), exc_info=True)
RE_UNKNOW_METRICS = re.compile("Unknown metrics: (.*) \(HTTP 400\)")
RE_UNKNOW_METRICS_LIST = re.compile("([^/ ,]*)/([^,]*)")
def batch_measures(self, measures, resource_infos, stats): def batch_measures(self, measures, resource_infos, stats):
# NOTE(sileht): We don't care about error here, we want # NOTE(sileht): We don't care about error here, we want
# resources metadata always been updated # resources metadata always been updated
try: try:
self._gnocchi.metric.batch_resources_metrics_measures(measures) self._gnocchi.metric.batch_resources_metrics_measures(
measures, create_metrics=True)
except gnocchi_exc.BadRequest as e: except gnocchi_exc.BadRequest as e:
m = self.RE_UNKNOW_METRICS.match(six.text_type(e)) if not isinstance(e.message, dict):
if m is None: raise
if e.message.get('cause') != 'Unknown resources':
raise raise
# NOTE(sileht): Create all missing resources and metrics for gnocchi_id in e.message['detail']:
metric_list = self.RE_UNKNOW_METRICS_LIST.findall(m.group(1))
gnocchi_ids_freshly_handled = set()
for gnocchi_id, metric_name in metric_list:
if gnocchi_id in gnocchi_ids_freshly_handled:
continue
resource = resource_infos[gnocchi_id]['resource'] resource = resource_infos[gnocchi_id]['resource']
resource_type = resource_infos[gnocchi_id]['resource_type'] resource_type = resource_infos[gnocchi_id]['resource_type']
try: try:
self._if_not_cached("create", resource_type, resource, self._if_not_cached("create", resource_type, resource,
self._create_resource) self._create_resource)
except gnocchi_exc.ResourceAlreadyExists: except gnocchi_exc.ResourceAlreadyExists:
metric = {'resource_id': resource['id'], # NOTE(sileht): resource created in the meantime
'name': metric_name} pass
metric.update(resource["metrics"][metric_name])
try:
self._gnocchi.metric.create(metric)
except gnocchi_exc.NamedMetricAlreadyExists:
# NOTE(sileht): metric created in the meantime
pass
except gnocchi_exc.ClientException as e:
LOG.error(six.text_type(e))
# We cannot post measures for this metric
del measures[gnocchi_id][metric_name]
if not measures[gnocchi_id]:
del measures[gnocchi_id]
except gnocchi_exc.ClientException as e: except gnocchi_exc.ClientException as e:
LOG.error(six.text_type(e)) LOG.error(six.text_type(e))
# We cannot post measures for this resource # We cannot post measures for this resource
del measures[gnocchi_id] del measures[gnocchi_id]
gnocchi_ids_freshly_handled.add(gnocchi_id)
else:
gnocchi_ids_freshly_handled.add(gnocchi_id)
# NOTE(sileht): we have created missing resources/metrics, # NOTE(sileht): we have created missing resources/metrics,
# now retry to post measures # now retry to post measures
self._gnocchi.metric.batch_resources_metrics_measures(measures) self._gnocchi.metric.batch_resources_metrics_measures(
measures, create_metrics=True)
# FIXME(sileht): take care of measures removed in stats # FIXME(sileht): take care of measures removed in stats
LOG.debug("%(measures)d measures posted against %(metrics)d " LOG.debug("%(measures)d measures posted against %(metrics)d "

39
ceilometer/tests/unit/dispatcher/test_gnocchi.py
View File

@ -452,10 +452,9 @@ class DispatcherWorkflowTest(base.BaseTestCase,
] ]
default_workflow = dict(resource_exists=True, default_workflow = dict(resource_exists=True,
metric_exists=True,
post_measure_fail=False, post_measure_fail=False,
create_resource_fail=False, create_resource_fail=False,
create_metric_fail=False, create_resource_race=False,
update_resource_fail=False, update_resource_fail=False,
retry_post_measures_fail=False) retry_post_measures_fail=False)
workflow_scenarios = [ workflow_scenarios = [
@ -463,10 +462,9 @@ class DispatcherWorkflowTest(base.BaseTestCase,
('new_resource', dict(resource_exists=False)), ('new_resource', dict(resource_exists=False)),
('new_resource_fail', dict(resource_exists=False, ('new_resource_fail', dict(resource_exists=False,
create_resource_fail=True)), create_resource_fail=True)),
('new_resource_race', dict(resource_exists=False,
create_resource_race=True)),
('resource_update_fail', dict(update_resource_fail=True)), ('resource_update_fail', dict(update_resource_fail=True)),
('new_metric', dict(metric_exists=False)),
('new_metric_fail', dict(metric_exists=False,
create_metric_fail=True)),
('retry_fail', dict(resource_exists=False, ('retry_fail', dict(resource_exists=False,
retry_post_measures_fail=True)), retry_post_measures_fail=True)),
('measure_fail', dict(post_measure_fail=True)), ('measure_fail', dict(post_measure_fail=True)),
@ -572,7 +570,8 @@ class DispatcherWorkflowTest(base.BaseTestCase,
expected_calls = [ expected_calls = [
mock.call.capabilities.list(), mock.call.capabilities.list(),
mock.call.metric.batch_resources_metrics_measures( mock.call.metric.batch_resources_metrics_measures(
{gnocchi_id: {metric_name: self.measures_attributes}}) {gnocchi_id: {metric_name: self.measures_attributes}},
create_metrics=True)
] ]
expected_debug = [ expected_debug = [
mock.call('gnocchi project found: %s', mock.call('gnocchi project found: %s',
@ -583,11 +582,11 @@ class DispatcherWorkflowTest(base.BaseTestCase,
batch_side_effect = [] batch_side_effect = []
if self.post_measure_fail: if self.post_measure_fail:
batch_side_effect += [Exception('boom!')] batch_side_effect += [Exception('boom!')]
elif not self.resource_exists or not self.metric_exists: elif not self.resource_exists:
batch_side_effect += [ batch_side_effect += [
gnocchi_exc.BadRequest( gnocchi_exc.BadRequest(
400, "Unknown metrics: %s/%s" % (gnocchi_id, 400, {"cause": "Unknown resources",
metric_name))] 'detail': [gnocchi_id]})]
attributes = self.postable_attributes.copy() attributes = self.postable_attributes.copy()
attributes.update(self.patchable_attributes) attributes.update(self.patchable_attributes)
attributes['id'] = self.sample['resource_id'] attributes['id'] = self.sample['resource_id']
@ -605,30 +604,18 @@ class DispatcherWorkflowTest(base.BaseTestCase,
if self.create_resource_fail: if self.create_resource_fail:
fakeclient.resource.create.side_effect = [Exception('boom!')] fakeclient.resource.create.side_effect = [Exception('boom!')]
elif self.resource_exists: elif self.create_resource_race:
fakeclient.resource.create.side_effect = [ fakeclient.resource.create.side_effect = [
gnocchi_exc.ResourceAlreadyExists(409)] gnocchi_exc.ResourceAlreadyExists(409)]
expected_calls.append(mock.call.metric.create({
'name': self.sample['counter_name'],
'unit': self.sample['counter_unit'],
'resource_id': resource_id}))
if self.create_metric_fail:
fakeclient.metric.create.side_effect = [Exception('boom!')]
elif self.metric_exists:
fakeclient.metric.create.side_effect = [
gnocchi_exc.NamedMetricAlreadyExists(409)]
else:
fakeclient.metric.create.side_effect = [None]
else: # not resource_exists else: # not resource_exists
expected_debug.append(mock.call( expected_debug.append(mock.call(
'Resource %s created', self.sample['resource_id'])) 'Resource %s created', self.sample['resource_id']))
if not self.create_resource_fail and not self.create_metric_fail: if not self.create_resource_fail:
expected_calls.append( expected_calls.append(
mock.call.metric.batch_resources_metrics_measures( mock.call.metric.batch_resources_metrics_measures(
{gnocchi_id: {metric_name: self.measures_attributes}}) {gnocchi_id: {metric_name: self.measures_attributes}},
create_metrics=True)
) )
if self.retry_post_measures_fail: if self.retry_post_measures_fail:
@ -664,7 +651,7 @@ class DispatcherWorkflowTest(base.BaseTestCase,
self.dispatcher.record_metering_data([self.sample]) self.dispatcher.record_metering_data([self.sample])
# Check that the last log message is the expected one # Check that the last log message is the expected one
if (self.post_measure_fail or self.create_metric_fail if (self.post_measure_fail
or self.create_resource_fail or self.create_resource_fail
or self.retry_post_measures_fail or self.retry_post_measures_fail
or (self.update_resource_fail and self.patchable_attributes)): or (self.update_resource_fail and self.patchable_attributes)):