Allow reaggregation method to be specified in the gnocchi collector
This introduces a new option to the gnocchi collector: "re_aggregation_method". It allows to specify an aggregation method that's different from the retrieved aggregate type for dynamic aggregation. Work items: * Added a "re_aggregation_method" to the gnocchi collector * Updated the documentation * Added some unit tests Change-Id: Id176c99a8cfc7761ba2a67c89a5521d23505ecb5
This commit is contained in:
parent
df1b5530ae
commit
8a0f80ad91
|
@ -83,6 +83,9 @@ GNOCCHI_EXTRA_SCHEMA = {
|
|||
Required('resource_key', default='id'): All(str, Length(min=1)),
|
||||
Required('aggregation_method', default='max'):
|
||||
In(['max', 'mean', 'min', 'rate:max', 'rate:mean', 'rate:min']),
|
||||
Required('re_aggregation_method', default=None):
|
||||
In([None, 'mean', 'median', 'std',
|
||||
'min', 'max', 'sum', 'var', 'count']),
|
||||
Required('force_granularity', default=0): All(int, Range(min=0)),
|
||||
},
|
||||
}
|
||||
|
@ -292,8 +295,12 @@ class GnocchiCollector(collector.BaseCollector):
|
|||
if q_filter:
|
||||
query_parameters.append(q_filter)
|
||||
|
||||
re_aggregation_method = extra_args['re_aggregation_method']
|
||||
if re_aggregation_method is None:
|
||||
re_aggregation_method = extra_args['aggregation_method']
|
||||
|
||||
# build aggregration operation
|
||||
op = ["aggregate", extra_args['aggregation_method'],
|
||||
op = ["aggregate", re_aggregation_method,
|
||||
["metric", metric_name, extra_args['aggregation_method']]]
|
||||
|
||||
# get groupby
|
||||
|
|
|
@ -13,7 +13,11 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
#
|
||||
import datetime
|
||||
|
||||
from dateutil import tz
|
||||
import mock
|
||||
|
||||
from cloudkitty.collector import gnocchi
|
||||
from cloudkitty import tests
|
||||
from cloudkitty.tests import samples
|
||||
|
@ -146,3 +150,67 @@ class GnocchiCollectorTest(tests.TestCase):
|
|||
lop='or')
|
||||
expected = {'or': ['dummy1', 'dummy2']}
|
||||
self.assertEqual(expected, actual)
|
||||
|
||||
|
||||
class GnocchiCollectorAggregationOperationTest(tests.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
super(GnocchiCollectorAggregationOperationTest, self).setUp()
|
||||
self.conf.set_override('collector', 'gnocchi', 'collect')
|
||||
self.start = datetime.datetime(2019, 1, 1, tzinfo=tz.UTC)
|
||||
self.end = datetime.datetime(2019, 1, 1, 1, tzinfo=tz.UTC)
|
||||
|
||||
def do_test(self, expected_op, extra_args=None):
|
||||
conf = {
|
||||
'metrics': {
|
||||
'metric_one': {
|
||||
'unit': 'GiB',
|
||||
'groupby': ['project_id'],
|
||||
'extra_args': extra_args if extra_args else {},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
coll = gnocchi.GnocchiCollector(period=3600, conf=conf)
|
||||
with mock.patch.object(coll._conn.aggregates, 'fetch') as fetch_mock:
|
||||
coll._fetch_metric('metric_one', self.start, self.end)
|
||||
fetch_mock.assert_called_once_with(
|
||||
expected_op,
|
||||
groupby=['project_id', 'id'],
|
||||
resource_type='resource_x',
|
||||
search={'=': {'type': 'resource_x'}},
|
||||
start=self.start, stop=self.end,
|
||||
)
|
||||
|
||||
def test_no_agg_no_re_agg(self):
|
||||
extra_args = {'resource_type': 'resource_x'}
|
||||
expected_op = ["aggregate", "max", ["metric", "metric_one", "max"]]
|
||||
self.do_test(expected_op, extra_args=extra_args)
|
||||
|
||||
def test_custom_agg_no_re_agg(self):
|
||||
extra_args = {
|
||||
'resource_type': 'resource_x',
|
||||
'aggregation_method': 'mean',
|
||||
}
|
||||
expected_op = ["aggregate", "mean", ["metric", "metric_one", "mean"]]
|
||||
self.do_test(expected_op, extra_args=extra_args)
|
||||
|
||||
def test_no_agg_custom_re_agg(self):
|
||||
extra_args = {
|
||||
'resource_type': 'resource_x',
|
||||
're_aggregation_method': 'sum',
|
||||
}
|
||||
expected_op = ["aggregate", "sum", ["metric", "metric_one", "max"]]
|
||||
self.do_test(expected_op, extra_args=extra_args)
|
||||
|
||||
def test_custom_agg_custom_re_agg(self):
|
||||
extra_args = {
|
||||
'resource_type': 'resource_x',
|
||||
'aggregation_method': 'rate:mean',
|
||||
're_aggregation_method': 'sum',
|
||||
}
|
||||
expected_op = [
|
||||
"aggregate", "sum",
|
||||
["metric", "metric_one", "rate:mean"],
|
||||
]
|
||||
self.do_test(expected_op, extra_args=extra_args)
|
||||
|
|
|
@ -70,6 +70,7 @@ class MetricConfigValidationTest(tests.TestCase):
|
|||
expected_output['metric_one']['groupby'] += ['project_id', 'id']
|
||||
expected_output['metric_one']['extra_args'] = {
|
||||
'aggregation_method': 'max',
|
||||
're_aggregation_method': None,
|
||||
'force_granularity': 0,
|
||||
'resource_type': 'res',
|
||||
'resource_key': 'id',
|
||||
|
|
|
@ -244,6 +244,23 @@ specified. The extra args for each collector are detailed below.
|
|||
Gnocchi
|
||||
~~~~~~~
|
||||
|
||||
.. note:: In order to retrieve metrics from Gnocchi, Cloudkitty uses the
|
||||
dynamic aggregates endpoint. It builds an operation of the following
|
||||
format: ``(aggregate RE_AGGREGATION_METHOD (metric METRIC_NAME
|
||||
AGGREGATION_METHOD))``. This means "retrieve all aggregates of type
|
||||
``AGGREGATION_METHOD`` for the metric named ``METRIC_NAME`` and
|
||||
re-aggregate them using ``RE_AGGREGATION_METHOD``".
|
||||
|
||||
By default, the re-aggregation method defaults to the
|
||||
aggregation method.
|
||||
|
||||
Setting the re-aggregation method to a different value than the
|
||||
aggregation method is useful when the granularity of the aggregates
|
||||
does not match CloudKitty's collect period, or when using
|
||||
``rate:`` aggregation, as you're probably don't want a rate of rates,
|
||||
but rather a sum or max of rates.
|
||||
|
||||
|
||||
* ``resource_type``: No default value. The resource type the current metric is
|
||||
bound to.
|
||||
|
||||
|
@ -253,7 +270,10 @@ Gnocchi
|
|||
|
||||
* ``aggregation_method``: Defaults to ``max``. The aggregation method to use
|
||||
when retrieving measures from gnocchi. Must be one of ``min``, ``max``,
|
||||
``mean``.
|
||||
``mean``, ``rate:min``, ``rate:max``, ``rate:mean``.
|
||||
|
||||
* ``re_aggregation_method``: Defaults to ``aggregation_method``. The
|
||||
re_aggregation method to use when retrieving measures from gnocchi.
|
||||
|
||||
* ``force_granularity``: Defaults to ``0``. If > 0, this granularity will be
|
||||
used for metric aggregations. Else, the lowest available granularity will be
|
||||
|
|
|
@ -0,0 +1,7 @@
|
|||
---
|
||||
features:
|
||||
- |
|
||||
It is now possible to differentiate the aggregation method from the
|
||||
aggregate type in the gnocchi collector, in case the retrieved aggregates
|
||||
need to be re-aggregated. This has been introduced with the
|
||||
``re_aggregation_method`` option.
|
|
@ -1,6 +1,6 @@
|
|||
---
|
||||
security:
|
||||
- |
|
||||
Data filtering on the ``GET /v1/dataframes`` and `` GET /v2/dataframes``
|
||||
has been fixed. It was previously possible for users to retrieve data
|
||||
from other scopes through these endpoints.
|
||||
Data filtering on the ``GET /v1/dataframes`` and ``GET /v2/dataframes``
|
||||
endpoints has been fixed. It was previously possible for users to retrieve
|
||||
data from other scopes through these endpoints.
|
||||
|
|
Loading…
Reference in New Issue