diff --git a/cloudkitty/collector/gnocchi.py b/cloudkitty/collector/gnocchi.py index a4fbdc64..ceedaa7d 100644 --- a/cloudkitty/collector/gnocchi.py +++ b/cloudkitty/collector/gnocchi.py @@ -98,6 +98,16 @@ GNOCCHI_EXTRA_SCHEMA = { } +class AssociatedResourceNotFound(Exception): + """Exception raised when no resource can be associated with a metric.""" + + def __init__(self, resource_key, resource_id): + super(AssociatedResourceNotFound, self).__init__( + 'Resource with {}={} could not be found'.format( + resource_key, resource_id), + ) + + class GnocchiCollector(collector.BaseCollector): collector_name = 'gnocchi' @@ -227,12 +237,18 @@ class GnocchiCollector(collector.BaseCollector): # Get gnocchi specific conf extra_args = self.conf[metric_name]['extra_args'] - # Build query - query_parameters = self._generate_time_filter(start, end) - resource_type = extra_args['resource_type'] scope_key = CONF.collect.scope_key + # Build query + + # FIXME(peschk_l): In order not to miss any resource whose metrics may + # contain measures after its destruction, we scan resources over three + # collect periods. + start -= CONF.collect.period + end += CONF.collect.period + query_parameters = self._generate_time_filter(start, end) + if project_id: kwargs = {scope_key: project_id} query_parameters.append(self.gen_filter(**kwargs)) @@ -312,8 +328,12 @@ class GnocchiCollector(collector.BaseCollector): # metadata as defined in the conf metadata = dict() if resources_info is not None: - resource = resources_info[ - groupby[metconf['extra_args']['resource_key']]] + resource_key = metconf['extra_args']['resource_key'] + resource_id = groupby[resource_key] + try: + resource = resources_info[resource_id] + except KeyError: + raise AssociatedResourceNotFound(resource_key, resource_id) for i in metconf['metadata']: metadata[i] = resource.get(i, '') qty = data['measures']['measures']['aggregated'][0][2] @@ -348,8 +368,19 @@ class GnocchiCollector(collector.BaseCollector): for d in data: # Only if aggregates have been found if d['measures']['measures']['aggregated']: - metadata, groupby, qty = self._format_data( - met, d, resources_info) + try: + metadata, groupby, qty = self._format_data( + met, d, resources_info) + except AssociatedResourceNotFound as e: + LOG.warning( + '[{}] An error occured during data collection ' + 'between {} and {}: {}'.format( + project_id, + ck_utils.ts2dt(start), + ck_utils.ts2dt(end), + e), + ) + continue data = self.t_cloudkitty.format_item( groupby, metadata, diff --git a/cloudkitty/tests/collectors/test_gnocchi.py b/cloudkitty/tests/collectors/test_gnocchi.py index 6bf1b03f..9f6b5a0f 100644 --- a/cloudkitty/tests/collectors/test_gnocchi.py +++ b/cloudkitty/tests/collectors/test_gnocchi.py @@ -17,13 +17,33 @@ from cloudkitty.collector import gnocchi from cloudkitty import tests from cloudkitty.tests import samples +from cloudkitty import transformer class GnocchiCollectorTest(tests.TestCase): def setUp(self): super(GnocchiCollectorTest, self).setUp() self._tenant_id = samples.TENANT - self.collector = gnocchi.GnocchiCollector + self.conf.set_override('collector', 'gnocchi', 'collect') + self.conf.set_override( + 'gnocchi_auth_type', 'basic', 'collector_gnocchi') + + self.collector = gnocchi.GnocchiCollector( + transformer.get_transformers(), + period=3600, + conf=samples.DEFAULT_METRICS_CONF, + ) + + def test_format_data_raises_exception(self): + metconf = {'extra_args': {'resource_key': 'id'}} + data = {'group': {'id': '281b9dc6-5d02-4610-af2d-10d0d6887f48'}} + self.assertRaises( + gnocchi.AssociatedResourceNotFound, + self.collector._format_data, + metconf, + data, + resources_info={}, + ) # Filter generation def test_generate_one_field_filter(self): diff --git a/releasenotes/notes/fix-gnocchi-metadata-collection-74665e862483a383.yaml b/releasenotes/notes/fix-gnocchi-metadata-collection-74665e862483a383.yaml new file mode 100644 index 00000000..09c4d9b1 --- /dev/null +++ b/releasenotes/notes/fix-gnocchi-metadata-collection-74665e862483a383.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Metadata collection failures in the gnocchi collector caused by resources + having measures in periods where they are supposed to be deleted have + been fixed. Metadata is now collected over a three-period window in the + gnocchi collector.