openstack/cloudkitty
Code Issues Proposed changes

Allows multiple rating types for same metric for gnocchi

Browse Source 
Problem description
===================

It is not possible to create multiple rating types
for the same metric in Gnocchi, which forces operators
to create multiple metrics for the same resource type in
Gnocchi to create different rating types in Cloudkitty for
the same resource type in Gnocchi.

Proposal
========

We propose to extend the Gnocchi collector to allow operators
to create multiple rating types for the same metric in Gnocchi.
Using this approach we can create, for example, a rating type
for software licenses in a running instance and another rating
type for the instance flavor; it can be implemented using only
one metric in Gnocchi which has the instance installed softwares
and flavor metadata.

Change-Id: I69d4ba14cc72ba55e47baa6fd372f2085e1124da
This commit is contained in:
Pedro Henrique 2022-10-13 19:17:36 +00:00 committed by Pedro Henrique Pereira Martins
parent addb4d6aa8
commit 7966437397
6 changed files with 152 additions and 23 deletions

47
cloudkitty/collector/gnocchi.py

@ -13,6 +13,7 @@
# License for the specific language governing permissions and limitations
# under the License.
#
import copy
from datetime import timedelta
import requests
@ -24,6 +25,7 @@ from oslo_config import cfg
from oslo_log import log as logging
from voluptuous import All
from voluptuous import In
from voluptuous import Invalid
from voluptuous import Length
from voluptuous import Range
from voluptuous import Required
@ -39,6 +41,24 @@ LOG = logging.getLogger(__name__)
COLLECTOR_GNOCCHI_OPTS = 'collector_gnocchi'
def GnocchiMetricDict(value):
if isinstance(value, dict) and len(value.keys()) > 0:
return value
if isinstance(value, list) and len(value) > 0:
for v in value:
if not (isinstance(v, dict) and len(v.keys()) > 0):
raise Invalid("Not a dict with at least one key or a "
"list with at least one dict with at "
"least one key. Provided value: %s" % value)
return value
raise Invalid("Not a dict with at least one key or a "
"list with at least one dict with at "
"least one key. Provided value: %s" % value)
GNOCCHI_CONF_SCHEMA = {Required('metrics'): GnocchiMetricDict}
collector_gnocchi_opts = [
cfg.StrOpt(
'gnocchi_auth_type',
@ -183,16 +203,30 @@ class GnocchiCollector(collector.BaseCollector):
"""Check metrics configuration
"""
conf = collector.BaseCollector.check_configuration(conf)
conf = Schema(GNOCCHI_CONF_SCHEMA)(conf)
conf = copy.deepcopy(conf)
scope_key = CONF.collect.scope_key
metric_schema = Schema(collector.METRIC_BASE_SCHEMA).extend(
GNOCCHI_EXTRA_SCHEMA)
output = {}
for metric_name, metric in conf.items():
met = output[metric_name] = metric_schema(metric)
for metric_name, metric in conf['metrics'].items():
if not isinstance(metric, list):
metric = [metric]
for m in metric:
met = metric_schema(m)
m.update(met)
if scope_key not in m['groupby']:
m['groupby'].append(scope_key)
if met['extra_args']['resource_key'] not in m['groupby']:
m['groupby'].append(met['extra_args']['resource_key'])
if met['extra_args']['resource_key'] not in met['groupby']:
met['groupby'].append(met['extra_args']['resource_key'])
names = [metric_name]
alt_name = met.get('alt_name')
if alt_name is not None:
names.append(alt_name)
new_metric_name = "@#".join(names)
output[new_metric_name] = m
return output
@ -337,7 +371,7 @@ class GnocchiCollector(collector.BaseCollector):
if 'Metrics not found' not in e.message["cause"]:
raise
LOG.warning('[{scope}] Skipping this metric for the '
'current cycle.'.format(scope=project_id, err=e))
'current cycle.'.format(scope=project_id))
return []
def get_aggregation_api_arguments(self, end, metric_name, project_id,
@ -392,6 +426,7 @@ class GnocchiCollector(collector.BaseCollector):
@staticmethod
def generate_aggregation_operation(extra_args, metric_name):
metric_name = metric_name.split('@#')[0]
aggregation_method = extra_args['aggregation_method']
re_aggregation_method = aggregation_method

2
cloudkitty/orchestrator.py

@ -400,7 +400,7 @@ class Worker(BaseWorker):
return True
def do_execute_scope_processing(self, timestamp):
metrics = list(self._conf['metrics'].keys())
metrics = list(self._collector.conf.keys())
# Collection
metrics = sorted(metrics)
usage_data = self._do_collection(metrics, timestamp)

74
cloudkitty/tests/collectors/test_gnocchi.py

@ -67,6 +67,34 @@ class GnocchiCollectorTest(tests.TestCase):
}}]}
self.assertEqual(expected, actual)
def test_collector_retrieve_metrics(self):
expected_data = {"group": {"id": "id-1",
"revision_start": datetime.datetime(
2020, 1, 1, 1, 10, 0, tzinfo=tz.tzutc())
}}
data = [
{"group": {"id": "id-1", "revision_start": datetime.datetime(
2020, 1, 1, tzinfo=tz.tzutc())}},
expected_data
]
no_response = mock.patch(
'cloudkitty.collector.gnocchi.GnocchiCollector.fetch_all',
return_value=data,
)
for c in self.collector.conf:
with no_response:
actual_name, actual_data = self.collector.retrieve(
metric_name=c,
start=samples.FIRST_PERIOD_BEGIN,
end=samples.FIRST_PERIOD_END,
project_id=samples.TENANT,
q_filter=None,
)
def test_generate_two_fields_filter_different_operations(self):
actual = self.collector.gen_filter(
cop='>=',
@ -160,8 +188,8 @@ class GnocchiCollectorAggregationOperationTest(tests.TestCase):
self.start = datetime.datetime(2019, 1, 1, tzinfo=tz.tzutc())
self.end = datetime.datetime(2019, 1, 1, 1, tzinfo=tz.tzutc())
def do_test(self, expected_op, extra_args=None):
conf = {
def do_test(self, expected_op, extra_args=None, conf=None):
conf = conf or {
'metrics': {
'metric_one': {
'unit': 'GiB',
@ -172,16 +200,38 @@ class GnocchiCollectorAggregationOperationTest(tests.TestCase):
}
coll = gnocchi.GnocchiCollector(period=3600, conf=conf)
with mock.patch.object(coll._conn.aggregates, 'fetch') as fetch_mock:
coll._fetch_metric('metric_one', self.start, self.end)
fetch_mock.assert_called_once_with(
expected_op,
groupby=['project_id', 'id'],
resource_type='resource_x',
search={'=': {'type': 'resource_x'}},
start=self.start, stop=self.end,
granularity=3600
)
for c in coll.conf:
with mock.patch.object(coll._conn.aggregates,
'fetch') as fetch_mock:
coll._fetch_metric(c, self.start, self.end)
fetch_mock.assert_called_once_with(
expected_op,
groupby=['project_id', 'id'],
resource_type='resource_x',
search={'=': {'type': 'resource_x'}},
start=self.start, stop=self.end,
granularity=3600
)
def test_multiple_confs(self):
conf = {
'metrics': {
'metric_one': [{
'alt_name': 'foo',
'unit': 'GiB',
'groupby': ['project_id'],
'extra_args': {'resource_type': 'resource_x'},
}, {
'alt_name': 'bar',
'unit': 'GiB',
'groupby': ['project_id'],
'extra_args': {'resource_type': 'resource_x'},
}]
}
}
expected_op = ["aggregate", "max", ["metric", "metric_one", "max"]]
self.do_test(expected_op, conf=conf)
def test_no_agg_no_re_agg(self):
extra_args = {'resource_type': 'resource_x'}

18
cloudkitty/tests/test_orchestrator.py

@ -355,14 +355,19 @@ class WorkerTest(tests.TestCase):
self, update_scope_processing_state_db_mock,
persist_rating_data_mock, execute_measurements_rating_mock,
do_collection_mock):
self.worker._conf = {"metrics": {"metric1": "s", "metric2": "d"}}
self.worker._collector = collector.gnocchi.GnocchiCollector(
period=3600,
conf=tests.samples.DEFAULT_METRICS_CONF,
)
do_collection_mock.return_value = None
timestamp_now = tzutils.localized_now()
self.worker.do_execute_scope_processing(timestamp_now)
do_collection_mock.assert_has_calls([
mock.call(["metric1", "metric2"], timestamp_now)
mock.call(['cpu@#instance', 'image.size', 'ip.floating',
'network.incoming.bytes', 'network.outgoing.bytes',
'radosgw.objects.size', 'volume.size'], timestamp_now)
])
self.assertFalse(execute_measurements_rating_mock.called)
@ -378,7 +383,10 @@ class WorkerTest(tests.TestCase):
self, update_scope_processing_state_db_mock,
persist_rating_data_mock, execute_measurements_rating_mock,
do_collection_mock):
self.worker._conf = {"metrics": {"metric1": "s", "metric2": "d"}}
self.worker._collector = collector.gnocchi.GnocchiCollector(
period=3600,
conf=tests.samples.DEFAULT_METRICS_CONF,
)
usage_data_mock = {"some_usage_data": 2}
do_collection_mock.return_value = usage_data_mock
@ -391,7 +399,9 @@ class WorkerTest(tests.TestCase):
self.worker.do_execute_scope_processing(timestamp_now)
do_collection_mock.assert_has_calls([
mock.call(["metric1", "metric2"], timestamp_now)
mock.call(['cpu@#instance', 'image.size', 'ip.floating',
'network.incoming.bytes', 'network.outgoing.bytes',
'radosgw.objects.size', 'volume.size'], timestamp_now)
])
end_time = tzutils.add_delta(

27
doc/source/admin/configuration/collector.rst

@ -288,6 +288,33 @@ specified. The extra args for each collector are detailed below.
Gnocchi
~~~~~~~
Besides the common configuration, the Gnocchi collector also accepts a list of
rating types definitions for each metric. Using a list of rating types
definitions allows operators to rate different aspects of the same resource
type collected through the same metric in Gnocchi, otherwise operators would
need to create multiple metrics in Gnocchi to create multiple rating types in
CloudKitty.
.. code-block:: yaml
metrics:
instance.metric:
- unit: instance
alt_name: flavor
mutate: NUMBOOL
groupby:
- id
metadata:
- flavor_id
- unit: instance
alt_name: operating_system_license
mutate: NUMBOOL
groupby:
- id
metadata:
- os_license
.. note:: In order to retrieve metrics from Gnocchi, Cloudkitty uses the
dynamic aggregates endpoint. It builds an operation of the following
format: ``(aggregate RE_AGGREGATION_METHOD (metric METRIC_NAME

7
releasenotes/notes/allow-multiple-ranting-types-for-same-metric-in-gnocchi-1011ba2d5d36c073.yaml Normal file

@ -0,0 +1,7 @@
---
features:
- |
Extends the Gnocchi collector to allow operators
to create multiple rating types for the same metric in Gnocchi.
The Gnocchi collector is now accepting a list of configs for each
metric entry, instead of accepting only one configuration.