openstack/cloudkitty
Code Issues Proposed changes

Merge "Allows multiple rating types for same metric for gnocchi"

Browse Source
This commit is contained in:
Zuul 2023-06-12 14:35:27 +00:00 committed by Gerrit Code Review
commit f2e70ca36f
6 changed files with 152 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
cloudkitty/collector/gnocchi.py
View File

@ -13,6 +13,7 @@
# License for the specific language governing permissions and limitations
# under the License.
#
import copy
from datetime import timedelta
import requests
@ -24,6 +25,7 @@ from oslo_config import cfg
from oslo_log import log as logging
from voluptuous import All
from voluptuous import In
from voluptuous import Invalid
from voluptuous import Length
from voluptuous import Range
from voluptuous import Required
@ -39,6 +41,24 @@ LOG = logging.getLogger(__name__)
COLLECTOR_GNOCCHI_OPTS = 'collector_gnocchi'
def GnocchiMetricDict(value):
if isinstance(value, dict) and len(value.keys()) > 0:
return value
if isinstance(value, list) and len(value) > 0:
for v in value:
if not (isinstance(v, dict) and len(v.keys()) > 0):
raise Invalid("Not a dict with at least one key or a "
"list with at least one dict with at "
"least one key. Provided value: %s" % value)
return value
raise Invalid("Not a dict with at least one key or a "
"list with at least one dict with at "
"least one key. Provided value: %s" % value)
GNOCCHI_CONF_SCHEMA = {Required('metrics'): GnocchiMetricDict}
collector_gnocchi_opts = [
cfg.StrOpt(
'gnocchi_auth_type',
@ -183,16 +203,30 @@ class GnocchiCollector(collector.BaseCollector):
"""Check metrics configuration
"""
conf = collector.BaseCollector.check_configuration(conf)
conf = Schema(GNOCCHI_CONF_SCHEMA)(conf)
conf = copy.deepcopy(conf)
scope_key = CONF.collect.scope_key
metric_schema = Schema(collector.METRIC_BASE_SCHEMA).extend(
GNOCCHI_EXTRA_SCHEMA)
output = {}
for metric_name, metric in conf.items():
met = output[metric_name] = metric_schema(metric)
for metric_name, metric in conf['metrics'].items():
if not isinstance(metric, list):
metric = [metric]
for m in metric:
met = metric_schema(m)
m.update(met)
if scope_key not in m['groupby']:
m['groupby'].append(scope_key)
if met['extra_args']['resource_key'] not in m['groupby']:
m['groupby'].append(met['extra_args']['resource_key'])
if met['extra_args']['resource_key'] not in met['groupby']:
met['groupby'].append(met['extra_args']['resource_key'])
names = [metric_name]
alt_name = met.get('alt_name')
if alt_name is not None:
names.append(alt_name)
new_metric_name = "@#".join(names)
output[new_metric_name] = m
return output
@ -337,7 +371,7 @@ class GnocchiCollector(collector.BaseCollector):
if 'Metrics not found' not in e.message["cause"]:
raise
LOG.warning('[{scope}] Skipping this metric for the '
'current cycle.'.format(scope=project_id, err=e))
'current cycle.'.format(scope=project_id))
return []
def get_aggregation_api_arguments(self, end, metric_name, project_id,
@ -392,6 +426,7 @@ class GnocchiCollector(collector.BaseCollector):
@staticmethod
def generate_aggregation_operation(extra_args, metric_name):
metric_name = metric_name.split('@#')[0]
aggregation_method = extra_args['aggregation_method']
re_aggregation_method = aggregation_method

2
cloudkitty/orchestrator.py
View File

@ -400,7 +400,7 @@ class Worker(BaseWorker):
return True
def do_execute_scope_processing(self, timestamp):
metrics = list(self._conf['metrics'].keys())
metrics = list(self._collector.conf.keys())
# Collection
metrics = sorted(metrics)
usage_data = self._do_collection(metrics, timestamp)

74
cloudkitty/tests/collectors/test_gnocchi.py
View File

@ -67,6 +67,34 @@ class GnocchiCollectorTest(tests.TestCase):
}}]}
self.assertEqual(expected, actual)
def test_collector_retrieve_metrics(self):
expected_data = {"group": {"id": "id-1",
"revision_start": datetime.datetime(
2020, 1, 1, 1, 10, 0, tzinfo=tz.tzutc())
}}
data = [
{"group": {"id": "id-1", "revision_start": datetime.datetime(
2020, 1, 1, tzinfo=tz.tzutc())}},
expected_data
]
no_response = mock.patch(
'cloudkitty.collector.gnocchi.GnocchiCollector.fetch_all',
return_value=data,
)
for c in self.collector.conf:
with no_response:
actual_name, actual_data = self.collector.retrieve(
metric_name=c,
start=samples.FIRST_PERIOD_BEGIN,
end=samples.FIRST_PERIOD_END,
project_id=samples.TENANT,
q_filter=None,
)
def test_generate_two_fields_filter_different_operations(self):
actual = self.collector.gen_filter(
cop='>=',
@ -160,8 +188,8 @@ class GnocchiCollectorAggregationOperationTest(tests.TestCase):
self.start = datetime.datetime(2019, 1, 1, tzinfo=tz.tzutc())
self.end = datetime.datetime(2019, 1, 1, 1, tzinfo=tz.tzutc())
def do_test(self, expected_op, extra_args=None):
conf = {
def do_test(self, expected_op, extra_args=None, conf=None):
conf = conf or {
'metrics': {
'metric_one': {
'unit': 'GiB',
@ -172,16 +200,38 @@ class GnocchiCollectorAggregationOperationTest(tests.TestCase):
}
coll = gnocchi.GnocchiCollector(period=3600, conf=conf)
with mock.patch.object(coll._conn.aggregates, 'fetch') as fetch_mock:
coll._fetch_metric('metric_one', self.start, self.end)
fetch_mock.assert_called_once_with(
expected_op,
groupby=['project_id', 'id'],
resource_type='resource_x',
search={'=': {'type': 'resource_x'}},
start=self.start, stop=self.end,
granularity=3600
)
for c in coll.conf:
with mock.patch.object(coll._conn.aggregates,
'fetch') as fetch_mock:
coll._fetch_metric(c, self.start, self.end)
fetch_mock.assert_called_once_with(
expected_op,
groupby=['project_id', 'id'],
resource_type='resource_x',
search={'=': {'type': 'resource_x'}},
start=self.start, stop=self.end,
granularity=3600
)
def test_multiple_confs(self):
conf = {
'metrics': {
'metric_one': [{
'alt_name': 'foo',
'unit': 'GiB',
'groupby': ['project_id'],
'extra_args': {'resource_type': 'resource_x'},
}, {
'alt_name': 'bar',
'unit': 'GiB',
'groupby': ['project_id'],
'extra_args': {'resource_type': 'resource_x'},
}]
}
}
expected_op = ["aggregate", "max", ["metric", "metric_one", "max"]]
self.do_test(expected_op, conf=conf)
def test_no_agg_no_re_agg(self):
extra_args = {'resource_type': 'resource_x'}

18
cloudkitty/tests/test_orchestrator.py
View File

@ -355,14 +355,19 @@ class WorkerTest(tests.TestCase):
self, update_scope_processing_state_db_mock,
persist_rating_data_mock, execute_measurements_rating_mock,
do_collection_mock):
self.worker._conf = {"metrics": {"metric1": "s", "metric2": "d"}}
self.worker._collector = collector.gnocchi.GnocchiCollector(
period=3600,
conf=tests.samples.DEFAULT_METRICS_CONF,
)
do_collection_mock.return_value = None
timestamp_now = tzutils.localized_now()
self.worker.do_execute_scope_processing(timestamp_now)
do_collection_mock.assert_has_calls([
mock.call(["metric1", "metric2"], timestamp_now)
mock.call(['cpu@#instance', 'image.size', 'ip.floating',
'network.incoming.bytes', 'network.outgoing.bytes',
'radosgw.objects.size', 'volume.size'], timestamp_now)
])
self.assertFalse(execute_measurements_rating_mock.called)
@ -378,7 +383,10 @@ class WorkerTest(tests.TestCase):
self, update_scope_processing_state_db_mock,
persist_rating_data_mock, execute_measurements_rating_mock,
do_collection_mock):
self.worker._conf = {"metrics": {"metric1": "s", "metric2": "d"}}
self.worker._collector = collector.gnocchi.GnocchiCollector(
period=3600,
conf=tests.samples.DEFAULT_METRICS_CONF,
)
usage_data_mock = {"some_usage_data": 2}
do_collection_mock.return_value = usage_data_mock
@ -391,7 +399,9 @@ class WorkerTest(tests.TestCase):
self.worker.do_execute_scope_processing(timestamp_now)
do_collection_mock.assert_has_calls([
mock.call(["metric1", "metric2"], timestamp_now)
mock.call(['cpu@#instance', 'image.size', 'ip.floating',
'network.incoming.bytes', 'network.outgoing.bytes',
'radosgw.objects.size', 'volume.size'], timestamp_now)
])
end_time = tzutils.add_delta(

27
doc/source/admin/configuration/collector.rst
View File

@ -288,6 +288,33 @@ specified. The extra args for each collector are detailed below.
Gnocchi
~~~~~~~
Besides the common configuration, the Gnocchi collector also accepts a list of
rating types definitions for each metric. Using a list of rating types
definitions allows operators to rate different aspects of the same resource
type collected through the same metric in Gnocchi, otherwise operators would
need to create multiple metrics in Gnocchi to create multiple rating types in
CloudKitty.
.. code-block:: yaml
metrics:
instance.metric:
- unit: instance
alt_name: flavor
mutate: NUMBOOL
groupby:
- id
metadata:
- flavor_id
- unit: instance
alt_name: operating_system_license
mutate: NUMBOOL
groupby:
- id
metadata:
- os_license
.. note:: In order to retrieve metrics from Gnocchi, Cloudkitty uses the
dynamic aggregates endpoint. It builds an operation of the following
format: ``(aggregate RE_AGGREGATION_METHOD (metric METRIC_NAME

7
releasenotes/notes/allow-multiple-ranting-types-for-same-metric-in-gnocchi-1011ba2d5d36c073.yaml Normal file
View File

@ -0,0 +1,7 @@
---
features:
- |
Extends the Gnocchi collector to allow operators
to create multiple rating types for the same metric in Gnocchi.
The Gnocchi collector is now accepting a list of configs for each
metric entry, instead of accepting only one configuration.