Allow reaggregation method to be specified in the gnocchi collector

This introduces a new option to the gnocchi collector: "re_aggregation_method".
It allows to specify an aggregation method that's different from the
retrieved aggregate type for dynamic aggregation.

Work items:

* Added a "re_aggregation_method" to the gnocchi collector
* Updated the documentation
* Added some unit tests

Change-Id: Id176c99a8cfc7761ba2a67c89a5521d23505ecb5

cloudkitty/collector/gnocchi.py

@@ -83,6 +83,9 @@ GNOCCHI_EXTRA_SCHEMA = {
         Required('resource_key', default='id'): All(str, Length(min=1)),
         Required('aggregation_method', default='max'):
             In(['max', 'mean', 'min', 'rate:max', 'rate:mean', 'rate:min']),
+        Required('re_aggregation_method', default=None):
+            In([None, 'mean', 'median', 'std',
+                'min', 'max', 'sum', 'var', 'count']),
         Required('force_granularity', default=0): All(int, Range(min=0)),
     },
 }
@@ -292,8 +295,12 @@ class GnocchiCollector(collector.BaseCollector):
         if q_filter:
             query_parameters.append(q_filter)
 
+        re_aggregation_method = extra_args['re_aggregation_method']
+        if re_aggregation_method is None:
+            re_aggregation_method = extra_args['aggregation_method']
+
         # build aggregration operation
-        op = ["aggregate", extra_args['aggregation_method'],
+        op = ["aggregate", re_aggregation_method,
               ["metric", metric_name, extra_args['aggregation_method']]]
 
         # get groupby

cloudkitty/tests/collectors/test_gnocchi.py

@@ -13,7 +13,11 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 #
-#
+import datetime
+
+from dateutil import tz
+import mock
+
 from cloudkitty.collector import gnocchi
 from cloudkitty import tests
 from cloudkitty.tests import samples
@@ -146,3 +150,67 @@ class GnocchiCollectorTest(tests.TestCase):
             lop='or')
         expected = {'or': ['dummy1', 'dummy2']}
         self.assertEqual(expected, actual)
+
+
+class GnocchiCollectorAggregationOperationTest(tests.TestCase):
+
+    def setUp(self):
+        super(GnocchiCollectorAggregationOperationTest, self).setUp()
+        self.conf.set_override('collector', 'gnocchi', 'collect')
+        self.start = datetime.datetime(2019, 1, 1, tzinfo=tz.UTC)
+        self.end = datetime.datetime(2019, 1, 1, 1, tzinfo=tz.UTC)
+
+    def do_test(self, expected_op, extra_args=None):
+        conf = {
+            'metrics': {
+                'metric_one': {
+                    'unit': 'GiB',
+                    'groupby': ['project_id'],
+                    'extra_args': extra_args if extra_args else {},
+                }
+            }
+        }
+
+        coll = gnocchi.GnocchiCollector(period=3600, conf=conf)
+        with mock.patch.object(coll._conn.aggregates, 'fetch') as fetch_mock:
+            coll._fetch_metric('metric_one', self.start, self.end)
+            fetch_mock.assert_called_once_with(
+                expected_op,
+                groupby=['project_id', 'id'],
+                resource_type='resource_x',
+                search={'=': {'type': 'resource_x'}},
+                start=self.start, stop=self.end,
+            )
+
+    def test_no_agg_no_re_agg(self):
+        extra_args = {'resource_type': 'resource_x'}
+        expected_op = ["aggregate", "max", ["metric", "metric_one", "max"]]
+        self.do_test(expected_op, extra_args=extra_args)
+
+    def test_custom_agg_no_re_agg(self):
+        extra_args = {
+            'resource_type': 'resource_x',
+            'aggregation_method': 'mean',
+        }
+        expected_op = ["aggregate", "mean", ["metric", "metric_one", "mean"]]
+        self.do_test(expected_op, extra_args=extra_args)
+
+    def test_no_agg_custom_re_agg(self):
+        extra_args = {
+            'resource_type': 'resource_x',
+            're_aggregation_method': 'sum',
+        }
+        expected_op = ["aggregate", "sum", ["metric", "metric_one", "max"]]
+        self.do_test(expected_op, extra_args=extra_args)
+
+    def test_custom_agg_custom_re_agg(self):
+        extra_args = {
+            'resource_type': 'resource_x',
+            'aggregation_method': 'rate:mean',
+            're_aggregation_method': 'sum',
+        }
+        expected_op = [
+            "aggregate", "sum",
+            ["metric", "metric_one", "rate:mean"],
+        ]
+        self.do_test(expected_op, extra_args=extra_args)

cloudkitty/tests/collectors/test_validation.py

@@ -70,6 +70,7 @@ class MetricConfigValidationTest(tests.TestCase):
         expected_output['metric_one']['groupby'] += ['project_id', 'id']
         expected_output['metric_one']['extra_args'] = {
             'aggregation_method': 'max',
+            're_aggregation_method': None,
             'force_granularity': 0,
             'resource_type': 'res',
             'resource_key': 'id',

doc/source/admin/configuration/collector.rst

@@ -244,6 +244,23 @@ specified. The extra args for each collector are detailed below.
 Gnocchi
 ~~~~~~~
 
+.. note:: In order to retrieve metrics from Gnocchi, Cloudkitty uses the
+          dynamic aggregates endpoint. It builds an operation of the following
+          format: ``(aggregate RE_AGGREGATION_METHOD (metric METRIC_NAME
+          AGGREGATION_METHOD))``. This means "retrieve all aggregates of type
+          ``AGGREGATION_METHOD`` for the metric named ``METRIC_NAME`` and
+          re-aggregate them using ``RE_AGGREGATION_METHOD``".
+
+          By default, the re-aggregation method defaults to the
+          aggregation method.
+
+          Setting the re-aggregation method to a different value than the
+          aggregation method is useful when the granularity of the aggregates
+          does not match CloudKitty's collect period, or when using
+          ``rate:`` aggregation, as you're probably don't want a rate of rates,
+          but rather a sum or max of rates.
+
+
 * ``resource_type``: No default value. The resource type the current metric is
   bound to.
 
@@ -253,7 +270,10 @@ Gnocchi
 
 * ``aggregation_method``: Defaults to ``max``. The aggregation method to use
   when retrieving measures from gnocchi. Must be one of ``min``, ``max``,
-  ``mean``.
+  ``mean``, ``rate:min``, ``rate:max``, ``rate:mean``.
+
+* ``re_aggregation_method``: Defaults to ``aggregation_method``. The
+  re_aggregation method to use when retrieving measures from gnocchi.
 
 * ``force_granularity``: Defaults to ``0``. If > 0, this granularity will be
   used for metric aggregations. Else, the lowest available granularity will be

releasenotes/notes/add-re-aggregation-method-option-gnocchi-collector-249917a14c4fc721.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    It is now possible to differentiate the aggregation method from the
+    aggregate type in the gnocchi collector, in case the retrieved aggregates
+    need to be re-aggregated. This has been introduced with the
+    ``re_aggregation_method`` option.

releasenotes/notes/fix-dataframe-filtering-282cae643457bb8b.yaml

@@ -2,5 +2,5 @@
 security:
   - |
     Data filtering on the ``GET /v1/dataframes`` and ``GET /v2/dataframes``
-    has been fixed. It was previously possible for users to retrieve data
-    from other scopes through these endpoints.
+    endpoints has been fixed. It was previously possible for users to retrieve
+    data from other scopes through these endpoints.