From 8a0f80ad915e9dff5ab1ec244fdaae5682f6f195 Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Wed, 20 Nov 2019 11:43:22 +0100 Subject: [PATCH] Allow reaggregation method to be specified in the gnocchi collector This introduces a new option to the gnocchi collector: "re_aggregation_method". It allows to specify an aggregation method that's different from the retrieved aggregate type for dynamic aggregation. Work items: * Added a "re_aggregation_method" to the gnocchi collector * Updated the documentation * Added some unit tests Change-Id: Id176c99a8cfc7761ba2a67c89a5521d23505ecb5 --- cloudkitty/collector/gnocchi.py | 9 ++- cloudkitty/tests/collectors/test_gnocchi.py | 70 ++++++++++++++++++- .../tests/collectors/test_validation.py | 1 + doc/source/admin/configuration/collector.rst | 22 +++++- ...on-gnocchi-collector-249917a14c4fc721.yaml | 7 ++ ...-dataframe-filtering-282cae643457bb8b.yaml | 6 +- 6 files changed, 109 insertions(+), 6 deletions(-) create mode 100644 releasenotes/notes/add-re-aggregation-method-option-gnocchi-collector-249917a14c4fc721.yaml diff --git a/cloudkitty/collector/gnocchi.py b/cloudkitty/collector/gnocchi.py index 3f33e7d8..cb1f025e 100644 --- a/cloudkitty/collector/gnocchi.py +++ b/cloudkitty/collector/gnocchi.py @@ -83,6 +83,9 @@ GNOCCHI_EXTRA_SCHEMA = { Required('resource_key', default='id'): All(str, Length(min=1)), Required('aggregation_method', default='max'): In(['max', 'mean', 'min', 'rate:max', 'rate:mean', 'rate:min']), + Required('re_aggregation_method', default=None): + In([None, 'mean', 'median', 'std', + 'min', 'max', 'sum', 'var', 'count']), Required('force_granularity', default=0): All(int, Range(min=0)), }, } @@ -292,8 +295,12 @@ class GnocchiCollector(collector.BaseCollector): if q_filter: query_parameters.append(q_filter) + re_aggregation_method = extra_args['re_aggregation_method'] + if re_aggregation_method is None: + re_aggregation_method = extra_args['aggregation_method'] + # build aggregration operation - op = ["aggregate", extra_args['aggregation_method'], + op = ["aggregate", re_aggregation_method, ["metric", metric_name, extra_args['aggregation_method']]] # get groupby diff --git a/cloudkitty/tests/collectors/test_gnocchi.py b/cloudkitty/tests/collectors/test_gnocchi.py index 671f5ef1..67940613 100644 --- a/cloudkitty/tests/collectors/test_gnocchi.py +++ b/cloudkitty/tests/collectors/test_gnocchi.py @@ -13,7 +13,11 @@ # License for the specific language governing permissions and limitations # under the License. # -# +import datetime + +from dateutil import tz +import mock + from cloudkitty.collector import gnocchi from cloudkitty import tests from cloudkitty.tests import samples @@ -146,3 +150,67 @@ class GnocchiCollectorTest(tests.TestCase): lop='or') expected = {'or': ['dummy1', 'dummy2']} self.assertEqual(expected, actual) + + +class GnocchiCollectorAggregationOperationTest(tests.TestCase): + + def setUp(self): + super(GnocchiCollectorAggregationOperationTest, self).setUp() + self.conf.set_override('collector', 'gnocchi', 'collect') + self.start = datetime.datetime(2019, 1, 1, tzinfo=tz.UTC) + self.end = datetime.datetime(2019, 1, 1, 1, tzinfo=tz.UTC) + + def do_test(self, expected_op, extra_args=None): + conf = { + 'metrics': { + 'metric_one': { + 'unit': 'GiB', + 'groupby': ['project_id'], + 'extra_args': extra_args if extra_args else {}, + } + } + } + + coll = gnocchi.GnocchiCollector(period=3600, conf=conf) + with mock.patch.object(coll._conn.aggregates, 'fetch') as fetch_mock: + coll._fetch_metric('metric_one', self.start, self.end) + fetch_mock.assert_called_once_with( + expected_op, + groupby=['project_id', 'id'], + resource_type='resource_x', + search={'=': {'type': 'resource_x'}}, + start=self.start, stop=self.end, + ) + + def test_no_agg_no_re_agg(self): + extra_args = {'resource_type': 'resource_x'} + expected_op = ["aggregate", "max", ["metric", "metric_one", "max"]] + self.do_test(expected_op, extra_args=extra_args) + + def test_custom_agg_no_re_agg(self): + extra_args = { + 'resource_type': 'resource_x', + 'aggregation_method': 'mean', + } + expected_op = ["aggregate", "mean", ["metric", "metric_one", "mean"]] + self.do_test(expected_op, extra_args=extra_args) + + def test_no_agg_custom_re_agg(self): + extra_args = { + 'resource_type': 'resource_x', + 're_aggregation_method': 'sum', + } + expected_op = ["aggregate", "sum", ["metric", "metric_one", "max"]] + self.do_test(expected_op, extra_args=extra_args) + + def test_custom_agg_custom_re_agg(self): + extra_args = { + 'resource_type': 'resource_x', + 'aggregation_method': 'rate:mean', + 're_aggregation_method': 'sum', + } + expected_op = [ + "aggregate", "sum", + ["metric", "metric_one", "rate:mean"], + ] + self.do_test(expected_op, extra_args=extra_args) diff --git a/cloudkitty/tests/collectors/test_validation.py b/cloudkitty/tests/collectors/test_validation.py index 40d9a1cb..7b2cc5d6 100644 --- a/cloudkitty/tests/collectors/test_validation.py +++ b/cloudkitty/tests/collectors/test_validation.py @@ -70,6 +70,7 @@ class MetricConfigValidationTest(tests.TestCase): expected_output['metric_one']['groupby'] += ['project_id', 'id'] expected_output['metric_one']['extra_args'] = { 'aggregation_method': 'max', + 're_aggregation_method': None, 'force_granularity': 0, 'resource_type': 'res', 'resource_key': 'id', diff --git a/doc/source/admin/configuration/collector.rst b/doc/source/admin/configuration/collector.rst index fe4bc3ba..b689f945 100644 --- a/doc/source/admin/configuration/collector.rst +++ b/doc/source/admin/configuration/collector.rst @@ -244,6 +244,23 @@ specified. The extra args for each collector are detailed below. Gnocchi ~~~~~~~ +.. note:: In order to retrieve metrics from Gnocchi, Cloudkitty uses the + dynamic aggregates endpoint. It builds an operation of the following + format: ``(aggregate RE_AGGREGATION_METHOD (metric METRIC_NAME + AGGREGATION_METHOD))``. This means "retrieve all aggregates of type + ``AGGREGATION_METHOD`` for the metric named ``METRIC_NAME`` and + re-aggregate them using ``RE_AGGREGATION_METHOD``". + + By default, the re-aggregation method defaults to the + aggregation method. + + Setting the re-aggregation method to a different value than the + aggregation method is useful when the granularity of the aggregates + does not match CloudKitty's collect period, or when using + ``rate:`` aggregation, as you're probably don't want a rate of rates, + but rather a sum or max of rates. + + * ``resource_type``: No default value. The resource type the current metric is bound to. @@ -253,7 +270,10 @@ Gnocchi * ``aggregation_method``: Defaults to ``max``. The aggregation method to use when retrieving measures from gnocchi. Must be one of ``min``, ``max``, - ``mean``. + ``mean``, ``rate:min``, ``rate:max``, ``rate:mean``. + +* ``re_aggregation_method``: Defaults to ``aggregation_method``. The + re_aggregation method to use when retrieving measures from gnocchi. * ``force_granularity``: Defaults to ``0``. If > 0, this granularity will be used for metric aggregations. Else, the lowest available granularity will be diff --git a/releasenotes/notes/add-re-aggregation-method-option-gnocchi-collector-249917a14c4fc721.yaml b/releasenotes/notes/add-re-aggregation-method-option-gnocchi-collector-249917a14c4fc721.yaml new file mode 100644 index 00000000..195d4963 --- /dev/null +++ b/releasenotes/notes/add-re-aggregation-method-option-gnocchi-collector-249917a14c4fc721.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + It is now possible to differentiate the aggregation method from the + aggregate type in the gnocchi collector, in case the retrieved aggregates + need to be re-aggregated. This has been introduced with the + ``re_aggregation_method`` option. diff --git a/releasenotes/notes/fix-dataframe-filtering-282cae643457bb8b.yaml b/releasenotes/notes/fix-dataframe-filtering-282cae643457bb8b.yaml index 09f2b72f..82636b43 100644 --- a/releasenotes/notes/fix-dataframe-filtering-282cae643457bb8b.yaml +++ b/releasenotes/notes/fix-dataframe-filtering-282cae643457bb8b.yaml @@ -1,6 +1,6 @@ --- security: - | - Data filtering on the ``GET /v1/dataframes`` and `` GET /v2/dataframes`` - has been fixed. It was previously possible for users to retrieve data - from other scopes through these endpoints. + Data filtering on the ``GET /v1/dataframes`` and ``GET /v2/dataframes`` + endpoints has been fixed. It was previously possible for users to retrieve + data from other scopes through these endpoints.