openstack/ceilometer
Code Issues Proposed changes

simplify gnocchi batch push setup

Browse Source 
we run through samples and format samples and resources to what
gnocchi expects. this simplifies that process a little:
- don't needlessly sort and group by metric name
- build a full resource model iff resource needs to be created

Change-Id: I2ac25b3b0978eed664c500e645bae2d1b4ae6781
This commit is contained in:
gord chung 2017-11-25 18:01:03 -05:00 committed by gordon chung
parent 40317d7bae
commit 5b216fae74
1 changed files with 22 additions and 32 deletions

54
ceilometer/publisher/gnocchi.py

@ -292,8 +292,7 @@ class GnocchiPublisher(publisher.ConfigPublisherBase):
def publish_samples(self, data):
# NOTE(sileht): skip sample generated by gnocchi itself
data = [s for s in data if not self._is_gnocchi_activity(s)]
data.sort(key=lambda s: (s.resource_id, s.name))
data.sort(key=operator.attrgetter('resource_id'))
resource_grouped_samples = itertools.groupby(
data, key=operator.attrgetter('resource_id'))
@ -303,13 +302,8 @@ class GnocchiPublisher(publisher.ConfigPublisherBase):
# NOTE(sileht): / is forbidden by Gnocchi
resource_id = resource_id.replace('/', '_')
metric_grouped_samples = itertools.groupby(
list(samples_of_resource),
key=operator.attrgetter('name'))
res_info = {}
for metric_name, samples in metric_grouped_samples:
samples = list(samples)
for sample in samples_of_resource:
metric_name = sample.name
rd = self.metric_map.get(metric_name)
if rd is None:
if metric_name not in self._already_logged_metric_names:
@ -320,28 +314,22 @@ class GnocchiPublisher(publisher.ConfigPublisherBase):
if rd.cfg.get("ignore"):
continue
res_info['resource_type'] = rd.cfg['resource_type']
res_info.setdefault("resource", {}).update({
"id": resource_id,
"user_id": samples[0].user_id,
"project_id": samples[0].project_id,
"metrics": rd.metrics,
})
if resource_id not in gnocchi_data:
gnocchi_data[resource_id] = {
'resource_type': rd.cfg['resource_type'],
'resource': {"id": resource_id,
"user_id": sample.user_id,
"project_id": sample.project_id,
"metrics": rd.metrics}}
for sample in samples:
res_info.setdefault("resource_extra", {}).update(
rd.sample_attributes(sample))
m = measures.setdefault(resource_id, {}).setdefault(
metric_name, [])
m.append({'timestamp': sample.timestamp,
'value': sample.volume})
unit = sample.unit
metric = sample.name
res_info['resource']['metrics'][metric]['unit'] = unit
res_info["resource"].update(res_info["resource_extra"])
if res_info:
gnocchi_data[resource_id] = res_info
gnocchi_data[resource_id].setdefault(
"resource_extra", {}).update(rd.sample_attributes(sample))
measures.setdefault(resource_id, {}).setdefault(
metric_name, []).append({'timestamp': sample.timestamp,
'value': sample.volume})
# TODO(gordc): unit should really be part of metric definition
gnocchi_data[resource_id]['resource']['metrics'][
metric_name]['unit'] = sample.unit
try:
self.batch_measures(measures, gnocchi_data)
@ -369,7 +357,8 @@ class GnocchiPublisher(publisher.ConfigPublisherBase):
resource_ids = set([r['original_resource_id']
for r in e.message['detail']])
return [(resource_infos[rid]['resource_type'],
resource_infos[rid]['resource'])
resource_infos[rid]['resource'],
resource_infos[rid]['resource_extra'])
for rid in resource_ids]
def batch_measures(self, measures, resource_infos):
@ -385,8 +374,9 @@ class GnocchiPublisher(publisher.ConfigPublisherBase):
raise
resources = self._extract_resources_from_error(e, resource_infos)
for resource_type, resource in resources:
for resource_type, resource, resource_extra in resources:
try:
resource.update(resource_extra)
self._if_not_cached("create", resource_type, resource,
self._create_resource)
except gnocchi_exc.ResourceAlreadyExists: