diff --git a/cloudkitty/collector/gnocchi.py b/cloudkitty/collector/gnocchi.py index 7df254bf..da17d400 100644 --- a/cloudkitty/collector/gnocchi.py +++ b/cloudkitty/collector/gnocchi.py @@ -114,6 +114,21 @@ GNOCCHI_EXTRA_SCHEMA = { In(BASIC_AGGREGATION_METHODS), Required('force_granularity', default=3600): All(int, Range(min=0)), Required('use_all_resource_revisions', default=True): All(bool), + # Provide means for operators to customize the aggregation query + # executed against Gnocchi. By default we use the following: + # + # '(aggregate RE_AGGREGATION_METHOD + # (metric METRIC_NAME AGGREGATION_METHOD))' + # + # Therefore, this option enables operators to take full advantage of + # operations available in Gnocchi, such as any arithmetic operations, + # logical operations and many others. + # + # When using a custom aggregation query, you can keep the placeholders + # 'RE_AGGREGATION_METHOD', 'AGGREGATION_METHOD', and 'METRIC_NAME': + # they will be replaced at runtime by values from the metric + # configuration. + Required('custom_query', default=''): All(str), }, } @@ -369,9 +384,32 @@ class GnocchiCollector(collector.BaseCollector): def build_operation_command(self, metric_name): extra_args = self.conf[metric_name]['extra_args'] - re_aggregation_method = extra_args['re_aggregation_method'] + op = self.generate_aggregation_operation(extra_args, metric_name) + + LOG.debug("Aggregation operation [%s] used to retrieve metric [%s].", + op, metric_name) + return op + + @staticmethod + def generate_aggregation_operation(extra_args, metric_name): + aggregation_method = extra_args['aggregation_method'] + re_aggregation_method = aggregation_method + + if 're_aggregation_method' in extra_args: + re_aggregation_method = extra_args['re_aggregation_method'] + op = ["aggregate", re_aggregation_method, - ["metric", metric_name, extra_args['aggregation_method']]] + ["metric", metric_name, aggregation_method]] + + custom_gnocchi_query = extra_args.get('custom_query') + if custom_gnocchi_query: + LOG.debug("Using custom Gnocchi query [%s] with metric [%s].", + custom_gnocchi_query, metric_name) + op = custom_gnocchi_query.replace( + 'RE_AGGREGATION_METHOD', re_aggregation_method).replace( + 'AGGREGATION_METHOD', aggregation_method).replace( + 'METRIC_NAME', metric_name) + return op def _format_data(self, metconf, data, resources_info=None): diff --git a/cloudkitty/tests/collectors/test_gnocchi.py b/cloudkitty/tests/collectors/test_gnocchi.py index 6600ae82..9f16382e 100644 --- a/cloudkitty/tests/collectors/test_gnocchi.py +++ b/cloudkitty/tests/collectors/test_gnocchi.py @@ -262,3 +262,41 @@ class GnocchiCollectorAggregationOperationTest(tests.TestCase): data_filtered = list(data_filtered) self.assertEqual(1, len(data_filtered)) self.assertEqual(expected_data, data_filtered[0]) + + def test_generate_aggregation_operation_same_reaggregation(self): + metric_name = "test" + extra_args = {"aggregation_method": 'mean'} + + expected_op = ["aggregate", 'mean', ["metric", "test", 'mean']] + + op = gnocchi.GnocchiCollector.generate_aggregation_operation( + extra_args, metric_name) + + self.assertEqual(expected_op, op) + + def test_generate_aggregation_operation_different_reaggregation(self): + metric_name = "test" + extra_args = {"aggregation_method": 'mean', + "re_aggregation_method": 'max'} + + expected_op = ["aggregate", 'max', ["metric", "test", 'mean']] + + op = gnocchi.GnocchiCollector.generate_aggregation_operation( + extra_args, metric_name) + + self.assertEqual(expected_op, op) + + def test_generate_aggregation_operation_custom_query(self): + metric_name = "test" + extra_args = {"aggregation_method": 'mean', + "re_aggregation_method": 'max', + "custom_query": + "(* (aggregate RE_AGGREGATION_METHOD (metric " + "METRIC_NAME AGGREGATION_METHOD)) -1)"} + + expected_op = "(* (aggregate max (metric test mean)) -1)" + + op = gnocchi.GnocchiCollector.generate_aggregation_operation( + extra_args, metric_name) + + self.assertEqual(expected_op, op) diff --git a/cloudkitty/tests/collectors/test_validation.py b/cloudkitty/tests/collectors/test_validation.py index c62974f3..bd08526d 100644 --- a/cloudkitty/tests/collectors/test_validation.py +++ b/cloudkitty/tests/collectors/test_validation.py @@ -69,9 +69,13 @@ class MetricConfigValidationTest(tests.TestCase): expected_output = copy.deepcopy(self.base_output) expected_output['metric_one']['groupby'] += ['project_id', 'id'] expected_output['metric_one']['extra_args'] = { - 'aggregation_method': 'max', 're_aggregation_method': 'max', - 'force_granularity': 3600, 'resource_type': 'res', - 'resource_key': 'id', 'use_all_resource_revisions': True} + 'aggregation_method': 'max', + 're_aggregation_method': 'max', + 'force_granularity': 3600, + 'resource_type': 'res', + 'resource_key': 'id', + 'use_all_resource_revisions': True, + 'custom_query': ''} self.assertEqual( collector.gnocchi.GnocchiCollector.check_configuration(data), diff --git a/doc/source/admin/configuration/collector.rst b/doc/source/admin/configuration/collector.rst index e77c1c27..9291b631 100644 --- a/doc/source/admin/configuration/collector.rst +++ b/doc/source/admin/configuration/collector.rst @@ -1,3 +1,4 @@ + ========================= Collector configuration ========================= @@ -307,6 +308,24 @@ Gnocchi CloudKitty for a resource id. The default behavior is maintained, which means, CloudKitty always use all of the data points returned. +* ``custom_query``: Provide means for operators to customize the aggregation + query executed against Gnocchi. By default we use the following ``(aggregate + RE_AGGREGATION_METHOD (metric METRIC_NAME AGGREGATION_METHOD))``. Therefore, + this option enables operators to take full advantage of operations available + in Gnocchi such as any arithmetic operations, logical operations and many + others. When using a custom aggregation query, you can keep the placeholders + ``RE_AGGREGATION_METHOD``, ``AGGREGATION_METHOD``, and ``METRIC_NAME``: they + will be replaced at runtime by values from the metric configuration. + + One example use case is metrics that are supposed to be always growing + values, such as RadosGW usage data. The usage data is affected by usage data + trimming on RadosGW, which can lead to swaps (meaning, that the right side + value of the series is smaller than the left side value) in the data series + in Gnocchi. Therefore, to handle this situation one could, for instance, use + the following custom query: ``(div (+ (aggregate RE_AGGREGATION_METHOD + (metric METRIC_NAME AGGREGATION_METHOD)) (abs (aggregate + RE_AGGREGATION_METHOD (metric METRIC_NAME AGGREGATION_METHOD)))) 2)``: this + custom query would return ``0`` when the value of the series swap. Monasca ~~~~~~~ diff --git a/releasenotes/notes/custom-gnocchi-query-a391f5e83d55d771.yaml b/releasenotes/notes/custom-gnocchi-query-a391f5e83d55d771.yaml new file mode 100644 index 00000000..beb4a0d2 --- /dev/null +++ b/releasenotes/notes/custom-gnocchi-query-a391f5e83d55d771.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Enable using custom queries with the Gnocchi collector. This option enables + operators to take full advantage of the operations that are available on + Gnocchi such as any arithmetic operation, logical operation and many + others.