openstack/cloudkitty
Code Issues Proposed changes

Custom query Gnocchi collector

Browse Source 
This patch proposes a method for operators to customize the aggregation
query executed against Gnocchi. By default, we use the following query:

(aggregate RE_AGGREGATION_METHOD (metric METRIC_NAME AGGREGATION_METHOD))

Therefore, this option enables operators to take full advantage of
operations available in Gnocchi, such as any arithmetic operations,
logical operations and many others. When using a custom aggregation
query, one can use the placeholders `RE_AGGREGATION_METHOD`,
`AGGREGATION_METHOD`, and `METRIC_NAME`: they will be replaced at
runtime by values from the metric configuration.

Different use cases can be addressed with the use of custom queries such
as handling RadosGW usage data trimming, which causes a decrease in the
usage data values; Libvirt attach/detach of disks, migration of VMs,
start/stop of VMs, which will zero the usage data that are gathered by
Ceilometer compute, and many other use cases where one might desire a
more complex operation to be executed on the data before CloudKitty
rates it.

Change-Id: I3419075d6df165409cb1375ad11a5b3f7faa7471
This commit is contained in:
Pierre Riteau 2021-07-26 15:11:58 +02:00
parent ffccfc222c
commit bac1960330
5 changed files with 111 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
cloudkitty/collector/gnocchi.py
View File

@ -114,6 +114,21 @@ GNOCCHI_EXTRA_SCHEMA = {
In(BASIC_AGGREGATION_METHODS),
Required('force_granularity', default=3600): All(int, Range(min=0)),
Required('use_all_resource_revisions', default=True): All(bool),
# Provide means for operators to customize the aggregation query
# executed against Gnocchi. By default we use the following:
#
# '(aggregate RE_AGGREGATION_METHOD
# (metric METRIC_NAME AGGREGATION_METHOD))'
#
# Therefore, this option enables operators to take full advantage of
# operations available in Gnocchi, such as any arithmetic operations,
# logical operations and many others.
#
# When using a custom aggregation query, you can keep the placeholders
# 'RE_AGGREGATION_METHOD', 'AGGREGATION_METHOD', and 'METRIC_NAME':
# they will be replaced at runtime by values from the metric
# configuration.
Required('custom_query', default=''): All(str),
},
}
@ -369,9 +384,32 @@ class GnocchiCollector(collector.BaseCollector):
def build_operation_command(self, metric_name):
extra_args = self.conf[metric_name]['extra_args']
op = self.generate_aggregation_operation(extra_args, metric_name)
LOG.debug("Aggregation operation [%s] used to retrieve metric [%s].",
op, metric_name)
return op
@staticmethod
def generate_aggregation_operation(extra_args, metric_name):
aggregation_method = extra_args['aggregation_method']
re_aggregation_method = aggregation_method
if 're_aggregation_method' in extra_args:
re_aggregation_method = extra_args['re_aggregation_method']
op = ["aggregate", re_aggregation_method,
["metric", metric_name, extra_args['aggregation_method']]]
["metric", metric_name, aggregation_method]]
custom_gnocchi_query = extra_args.get('custom_query')
if custom_gnocchi_query:
LOG.debug("Using custom Gnocchi query [%s] with metric [%s].",
custom_gnocchi_query, metric_name)
op = custom_gnocchi_query.replace(
'RE_AGGREGATION_METHOD', re_aggregation_method).replace(
'AGGREGATION_METHOD', aggregation_method).replace(
'METRIC_NAME', metric_name)
return op
def _format_data(self, metconf, data, resources_info=None):

38
cloudkitty/tests/collectors/test_gnocchi.py
View File

@ -262,3 +262,41 @@ class GnocchiCollectorAggregationOperationTest(tests.TestCase):
data_filtered = list(data_filtered)
self.assertEqual(1, len(data_filtered))
self.assertEqual(expected_data, data_filtered[0])
def test_generate_aggregation_operation_same_reaggregation(self):
metric_name = "test"
extra_args = {"aggregation_method": 'mean'}
expected_op = ["aggregate", 'mean', ["metric", "test", 'mean']]
op = gnocchi.GnocchiCollector.generate_aggregation_operation(
extra_args, metric_name)
self.assertEqual(expected_op, op)
def test_generate_aggregation_operation_different_reaggregation(self):
metric_name = "test"
extra_args = {"aggregation_method": 'mean',
"re_aggregation_method": 'max'}
expected_op = ["aggregate", 'max', ["metric", "test", 'mean']]
op = gnocchi.GnocchiCollector.generate_aggregation_operation(
extra_args, metric_name)
self.assertEqual(expected_op, op)
def test_generate_aggregation_operation_custom_query(self):
metric_name = "test"
extra_args = {"aggregation_method": 'mean',
"re_aggregation_method": 'max',
"custom_query":
"(* (aggregate RE_AGGREGATION_METHOD (metric "
"METRIC_NAME AGGREGATION_METHOD)) -1)"}
expected_op = "(* (aggregate max (metric test mean)) -1)"
op = gnocchi.GnocchiCollector.generate_aggregation_operation(
extra_args, metric_name)
self.assertEqual(expected_op, op)

10
cloudkitty/tests/collectors/test_validation.py
View File

@ -69,9 +69,13 @@ class MetricConfigValidationTest(tests.TestCase):
expected_output = copy.deepcopy(self.base_output)
expected_output['metric_one']['groupby'] += ['project_id', 'id']
expected_output['metric_one']['extra_args'] = {
'aggregation_method': 'max', 're_aggregation_method': 'max',
'force_granularity': 3600, 'resource_type': 'res',
'resource_key': 'id', 'use_all_resource_revisions': True}
'aggregation_method': 'max',
're_aggregation_method': 'max',
'force_granularity': 3600,
'resource_type': 'res',
'resource_key': 'id',
'use_all_resource_revisions': True,
'custom_query': ''}
self.assertEqual(
collector.gnocchi.GnocchiCollector.check_configuration(data),

19
doc/source/admin/configuration/collector.rst
View File

@ -1,3 +1,4 @@
=========================
Collector configuration
=========================
@ -307,6 +308,24 @@ Gnocchi
CloudKitty for a resource id. The default behavior is maintained, which
means, CloudKitty always use all of the data points returned.
* ``custom_query``: Provide means for operators to customize the aggregation
query executed against Gnocchi. By default we use the following ``(aggregate
RE_AGGREGATION_METHOD (metric METRIC_NAME AGGREGATION_METHOD))``. Therefore,
this option enables operators to take full advantage of operations available
in Gnocchi such as any arithmetic operations, logical operations and many
others. When using a custom aggregation query, you can keep the placeholders
``RE_AGGREGATION_METHOD``, ``AGGREGATION_METHOD``, and ``METRIC_NAME``: they
will be replaced at runtime by values from the metric configuration.
One example use case is metrics that are supposed to be always growing
values, such as RadosGW usage data. The usage data is affected by usage data
trimming on RadosGW, which can lead to swaps (meaning, that the right side
value of the series is smaller than the left side value) in the data series
in Gnocchi. Therefore, to handle this situation one could, for instance, use
the following custom query: ``(div (+ (aggregate RE_AGGREGATION_METHOD
(metric METRIC_NAME AGGREGATION_METHOD)) (abs (aggregate
RE_AGGREGATION_METHOD (metric METRIC_NAME AGGREGATION_METHOD)))) 2)``: this
custom query would return ``0`` when the value of the series swap.
Monasca
~~~~~~~

7
releasenotes/notes/custom-gnocchi-query-a391f5e83d55d771.yaml Normal file
View File

@ -0,0 +1,7 @@
---
features:
- |
Enable using custom queries with the Gnocchi collector. This option enables
operators to take full advantage of the operations that are available on
Gnocchi such as any arithmetic operation, logical operation and many
others.