Manage metrics units in yaml configuration.
Allow users and operators to control the conversions units, by filling source and destinations units for the metrics. Differents units can now be affected to metrics from a same resource. Task: 6077 Story: 2001396 Change-Id: I79713063d530737df20d4d910afc7c7e489aa2c5
This commit is contained in:
MC
parent
c89a3435c5
commit
a0db1f138e
@ -85,12 +85,12 @@ class CeilometerCollector(collector.BaseCollector):
|
||||
|
||||
units_mappings = {
|
||||
'compute': 'instance',
|
||||
'image': 'MB',
|
||||
'volume': 'GB',
|
||||
'image': 'MiB',
|
||||
'volume': 'GiB',
|
||||
'network.bw.out': 'MB',
|
||||
'network.bw.in': 'MB',
|
||||
'network.floating': 'ip',
|
||||
'radosgw.usage': 'GB'
|
||||
'radosgw.usage': 'GiB'
|
||||
}
|
||||
|
||||
def __init__(self, transformers, **kwargs):
|
||||
@ -121,12 +121,13 @@ class CeilometerCollector(collector.BaseCollector):
|
||||
.get_metadata(resource_name))
|
||||
|
||||
try:
|
||||
info["unit"] = METRICS_CONF['services_units'][resource_name][1]
|
||||
tmp = METRICS_CONF['metrics_units'][resource_name]
|
||||
info['unit'] = list(tmp.values())[0]['unit']
|
||||
# NOTE(mc): deprecated except part kept for backward compatibility.
|
||||
except KeyError:
|
||||
LOG.warning('Error when trying to use yaml metrology conf.')
|
||||
LOG.warning('Fallback on the deprecated oslo config method.')
|
||||
info["unit"] = cls.units_mappings[resource_name]
|
||||
info['unit'] = cls.units_mappings[resource_name]
|
||||
|
||||
except KeyError:
|
||||
pass
|
||||
@ -217,9 +218,10 @@ class CeilometerCollector(collector.BaseCollector):
|
||||
instance_id)
|
||||
|
||||
try:
|
||||
met = list(METRICS_CONF['metrics_units']['compute'].values())
|
||||
compute_data.append(self.t_cloudkitty.format_item(
|
||||
instance,
|
||||
METRICS_CONF['services_units']['compute'],
|
||||
met[0]['unit'],
|
||||
1,
|
||||
))
|
||||
# NOTE(mc): deprecated except part kept for backward compatibility.
|
||||
@ -254,12 +256,24 @@ class CeilometerCollector(collector.BaseCollector):
|
||||
image = self._cacher.get_resource_detail('image',
|
||||
image_id)
|
||||
|
||||
image_size_mb = decimal.Decimal(image_stats.max) / units.Mi
|
||||
# Unit conversion
|
||||
try:
|
||||
conv_data = METRICS_CONF['metrics_units']['image']
|
||||
image_size_mb = ck_utils.convert_unit(
|
||||
decimal.Decimal(image_stats.max),
|
||||
conv_data['image.size'].get('factor', '1'),
|
||||
conv_data['image.size'].get('offset', '0'),
|
||||
)
|
||||
except KeyError:
|
||||
LOG.warning('Error when trying to use yaml metrology conf.')
|
||||
LOG.warning('Fallback on the deprecated hardcoded method.')
|
||||
image_size_mb = decimal.Decimal(image_stats.max) / units.Mi
|
||||
|
||||
try:
|
||||
met = list(METRICS_CONF['metrics_units']['image'].values())
|
||||
image_data.append(self.t_cloudkitty.format_item(
|
||||
image,
|
||||
METRICS_CONF['services_units']['image'],
|
||||
met[0]['unit'],
|
||||
image_size_mb,
|
||||
))
|
||||
# NOTE(mc): deprecated except part kept for backward compatibility.
|
||||
@ -296,10 +310,18 @@ class CeilometerCollector(collector.BaseCollector):
|
||||
volume = self._cacher.get_resource_detail('volume',
|
||||
volume_id)
|
||||
|
||||
# Unit conversion
|
||||
try:
|
||||
conv_data = METRICS_CONF['metrics_units']['volume']
|
||||
volume_stats.max = ck_utils.convert_unit(
|
||||
decimal.Decimal(volume_stats.max),
|
||||
conv_data['volume.size'].get('factor', '1'),
|
||||
conv_data['volume.size'].get('offset', '0'),
|
||||
)
|
||||
|
||||
volume_data.append(self.t_cloudkitty.format_item(
|
||||
volume,
|
||||
METRICS_CONF['services_units']['volume'],
|
||||
conv_data['volume.size']['unit'],
|
||||
volume_stats.max,
|
||||
))
|
||||
# NOTE(mc): deprecated except part kept for backward compatibility.
|
||||
@ -347,12 +369,24 @@ class CeilometerCollector(collector.BaseCollector):
|
||||
tap = self._cacher.get_resource_detail('network.tap',
|
||||
tap_id)
|
||||
|
||||
tap_bw_mb = decimal.Decimal(tap_stat.max) / units.M
|
||||
# Unit conversion
|
||||
try:
|
||||
conv = METRICS_CONF['metrics_units']['network.bw.' + direction]
|
||||
tap_bw_mb = ck_utils.convert_unit(
|
||||
decimal.Decimal(tap_stat.max),
|
||||
conv[resource_type].get('factor', '1'),
|
||||
conv[resource_type].get('offset', '0'),
|
||||
)
|
||||
except KeyError:
|
||||
LOG.warning('Error when trying to use yaml metrology conf.')
|
||||
LOG.warning('Fallback on the deprecated hardcoded method.')
|
||||
tap_bw_mb = decimal.Decimal(tap_stat.max) / units.M
|
||||
|
||||
try:
|
||||
met = METRICS_CONF['metrics_units']['network.bw.' + direction]
|
||||
bw_data.append(self.t_cloudkitty.format_item(
|
||||
tap,
|
||||
METRICS_CONF['services_units']['network.bw.' + direction],
|
||||
list(met.values())[0]['unit'],
|
||||
tap_bw_mb,
|
||||
))
|
||||
# NOTE(mc): deprecated except part kept for backward compatibility.
|
||||
@ -411,9 +445,10 @@ class CeilometerCollector(collector.BaseCollector):
|
||||
floating_id)
|
||||
|
||||
try:
|
||||
metric = METRICS_CONF['metrics_units']['network.floating']
|
||||
floating_data.append(self.t_cloudkitty.format_item(
|
||||
floating,
|
||||
METRICS_CONF['services_units']['network.floating'],
|
||||
list(metric.values())[0]['unit'],
|
||||
1,
|
||||
))
|
||||
# NOTE(mc): deprecated except part kept for backward compatibility.
|
||||
@ -448,9 +483,35 @@ class CeilometerCollector(collector.BaseCollector):
|
||||
raw_resource)
|
||||
self._cacher.add_resource_detail('radosgw.usage', rgw_id, rgw)
|
||||
rgw = self._cacher.get_resource_detail('radosgw.usage', rgw_id)
|
||||
rgw_size = decimal.Decimal(rgw_stats.max) / units.Gi
|
||||
rgw_data.append(self.t_cloudkitty.format_item(
|
||||
rgw, self.units_mappings["radosgw.usage"], rgw_size))
|
||||
|
||||
# Unit conversion
|
||||
try:
|
||||
conv_data = METRICS_CONF['metrics_units']['radosgw.usage']
|
||||
rgw_size = ck_utils.convert_unit(
|
||||
decimal.Decimal(rgw_stats.max),
|
||||
conv_data['radosgw.object.size'].get('factor', '1'),
|
||||
conv_data['radosgw.object.size'].get('offset', '0'),
|
||||
)
|
||||
|
||||
rgw_data.append(
|
||||
self.t_cloudkitty.format_item(
|
||||
rgw,
|
||||
conv_data['rados.objects.size']['unit'],
|
||||
rgw_size,
|
||||
)
|
||||
)
|
||||
except KeyError:
|
||||
LOG.warning('Error when trying to use yaml metrology conf.')
|
||||
LOG.warning('Fallback on the deprecated hardcoded method.')
|
||||
rgw_size = decimal.Decimal(rgw_stats.max) / units.Gi
|
||||
rgw_data.append(
|
||||
self.t_cloudkitty.format_item(
|
||||
rgw,
|
||||
self.units_mappings['radosgw.usage'],
|
||||
rgw_size,
|
||||
)
|
||||
)
|
||||
|
||||
if not rgw_data:
|
||||
raise collector.NoDataCollected(self.collector_name,
|
||||
'radosgw.usage')
|
||||
|
||||
@ -79,12 +79,12 @@ class GnocchiCollector(collector.BaseCollector):
|
||||
}
|
||||
units_mappings = {
|
||||
'compute': (1, 'instance'),
|
||||
'image': ('image.size', 'MB'),
|
||||
'volume': ('volume.size', 'GB'),
|
||||
'image': ('image.size', 'MiB'),
|
||||
'volume': ('volume.size', 'GiB'),
|
||||
'network.bw.out': ('network.outgoing.bytes', 'MB'),
|
||||
'network.bw.in': ('network.incoming.bytes', 'MB'),
|
||||
'network.floating': (1, 'ip'),
|
||||
'radosgw.usage': ('radosgw.objects.size', 'GB')
|
||||
'radosgw.usage': ('radosgw.objects.size', 'GiB')
|
||||
}
|
||||
default_unit = (1, 'unknown')
|
||||
|
||||
@ -114,12 +114,13 @@ class GnocchiCollector(collector.BaseCollector):
|
||||
.get_metadata(resource_name))
|
||||
|
||||
try:
|
||||
info["unit"] = METRICS_CONF['services_units'][resource_name][1]
|
||||
tmp = METRICS_CONF['metrics_units'][resource_name]
|
||||
info['unit'] = list(tmp.values())[0]['unit']
|
||||
# NOTE(mc): deprecated except part kept for backward compatibility.
|
||||
except KeyError:
|
||||
LOG.warning('Error when trying to use yaml metrology conf.')
|
||||
LOG.warning('Fallback on the deprecated oslo config method.')
|
||||
info["unit"] = cls.units_mappings[resource_name][1]
|
||||
info['unit'] = cls.units_mappings[resource_name][1]
|
||||
|
||||
except KeyError:
|
||||
pass
|
||||
@ -261,8 +262,9 @@ class GnocchiCollector(collector.BaseCollector):
|
||||
def resource_info(self, resource_name, start, end, project_id,
|
||||
q_filter=None):
|
||||
try:
|
||||
qty = METRICS_CONF['services_units'][resource_name].keys()[0]
|
||||
unit = METRICS_CONF['services_units'][resource_name].values()[0]
|
||||
tmp = METRICS_CONF['metrics_units'][resource_name]
|
||||
qty = list(tmp.keys())[0]
|
||||
unit = list(tmp.values())[0]['unit']
|
||||
# NOTE(mc): deprecated except part kept for backward compatibility.
|
||||
except KeyError:
|
||||
LOG.warning('Error when trying to use yaml metrology conf.')
|
||||
@ -290,20 +292,41 @@ class GnocchiCollector(collector.BaseCollector):
|
||||
|
||||
self._expand_metrics([resource_data], mappings, start, end)
|
||||
resource_data.pop('metrics', None)
|
||||
# Convert network.bw.in, network.bw.out and image unit to MB
|
||||
if resource.get('type') == 'instance_network_interface':
|
||||
resource_data[qty] = (
|
||||
decimal.Decimal(resource_data[qty]) / units.M)
|
||||
elif resource.get('type') == 'image':
|
||||
resource_data[qty] = (
|
||||
decimal.Decimal(resource_data[qty]) / units.Mi)
|
||||
elif resource.get('type') == 'ceph_account':
|
||||
resource_data[qty] = (
|
||||
decimal.Decimal(resource_data[qty]) / units.Gi)
|
||||
|
||||
# Unit conversion
|
||||
try:
|
||||
conv_data = METRICS_CONF['metrics_units'][resource_name][qty]
|
||||
if isinstance(qty, str):
|
||||
resource_data[qty] = ck_utils.convert_unit(
|
||||
resource_data[qty],
|
||||
conv_data.get('factor', '1'),
|
||||
conv_data.get('offset', '0'),
|
||||
)
|
||||
# NOTE(mc): deprecated except part kept for backward compatibility.
|
||||
except KeyError:
|
||||
LOG.warning('Error when trying to use yaml metrology conf.')
|
||||
LOG.warning('Fallback on the deprecated hardcoded method.')
|
||||
|
||||
if resource.get('type') == 'instance_network_interface':
|
||||
resource_data[qty] = (
|
||||
decimal.Decimal(resource_data[qty]) / units.M
|
||||
)
|
||||
elif resource.get('type') == 'image':
|
||||
resource_data[qty] = (
|
||||
decimal.Decimal(resource_data[qty]) / units.Mi
|
||||
)
|
||||
elif resource.get('type') == 'ceph_account':
|
||||
resource_data[qty] = (
|
||||
decimal.Decimal(resource_data[qty]) / units.Gi)
|
||||
|
||||
data = self.t_cloudkitty.format_item(
|
||||
resource_data, unit,
|
||||
resource_data,
|
||||
unit,
|
||||
decimal.Decimal(
|
||||
qty if isinstance(qty, int) else resource_data[qty]))
|
||||
qty if isinstance(qty, int) else resource_data[qty]
|
||||
)
|
||||
)
|
||||
|
||||
# NOTE(sheeprine): Reference to gnocchi resource used by storage
|
||||
data['resource_id'] = data['desc']['resource_id']
|
||||
formated_resources.append(data)
|
||||
|
||||
@ -85,8 +85,8 @@ class MonascaCollector(collector.BaseCollector):
|
||||
# or a decimal.Decimal object
|
||||
units_mappings = {
|
||||
'compute': (1, 'instance'),
|
||||
'image': ('image.size', 'MB'),
|
||||
'volume': ('volume.size', 'GB'),
|
||||
'image': ('image.size', 'MiB'),
|
||||
'volume': ('volume.size', 'GiB'),
|
||||
'network.bw.out': ('network.outgoing.bytes', 'MB'),
|
||||
'network.bw.in': ('network.incoming.bytes', 'MB'),
|
||||
'network.floating': (1, 'ip'),
|
||||
@ -133,7 +133,8 @@ class MonascaCollector(collector.BaseCollector):
|
||||
def _get_metadata(self, resource_type, transformers):
|
||||
info = {}
|
||||
try:
|
||||
info['unit'] = METRICS_CONF['services_units']
|
||||
met = list(METRICS_CONF['metrics_units'][resource_type].values())
|
||||
info['unit'] = met[0]['unit']
|
||||
# NOTE(mc): deprecated second try kept for backward compatibility.
|
||||
except KeyError:
|
||||
LOG.warning('Error when trying to use yaml metrology conf.')
|
||||
@ -268,24 +269,52 @@ class MonascaCollector(collector.BaseCollector):
|
||||
continue
|
||||
return resource_ids
|
||||
|
||||
def _expand_metrics(self, resource, resource_id, mappings, start, end):
|
||||
for name, statistics in mappings:
|
||||
qty = self._get_resource_qty(name, start,
|
||||
end, resource_id, statistics)
|
||||
if name in ['network.outgoing.bytes', 'network.incoming.bytes']:
|
||||
qty = qty / units.M
|
||||
elif 'image.' in name:
|
||||
qty = qty / units.Mi
|
||||
resource[name] = qty
|
||||
def _expand_metrics(self, resource, resource_id,
|
||||
mappings, start, end, resource_type):
|
||||
try:
|
||||
for name, statistics in mappings.items():
|
||||
qty = self._get_resource_qty(
|
||||
name,
|
||||
start,
|
||||
end,
|
||||
resource_id,
|
||||
statistics,
|
||||
)
|
||||
|
||||
conv_data = METRICS_CONF['metrics_units'][resource_type][name]
|
||||
resource[name] = ck_utils.convert_unit(
|
||||
qty,
|
||||
conv_data.get('factor', 1),
|
||||
conv_data.get('offset', 0),
|
||||
)
|
||||
# NOTE(mc): deprecated except part kept for backward compatibility.
|
||||
except KeyError:
|
||||
LOG.warning('Error when trying to use yaml metrology conf.')
|
||||
LOG.warning('Fallback on the deprecated hardcoded dict method.')
|
||||
|
||||
for name, statistics in mappings:
|
||||
qty = self._get_resource_qty(
|
||||
name,
|
||||
start,
|
||||
end,
|
||||
resource_id,
|
||||
statistics,
|
||||
)
|
||||
|
||||
names = ['network.outgoing.bytes', 'network.incoming.bytes']
|
||||
if name in names:
|
||||
qty = qty / units.M
|
||||
elif 'image.' in name:
|
||||
qty = qty / units.Mi
|
||||
resource[name] = qty
|
||||
|
||||
def resource_info(self, resource_type, start, end,
|
||||
project_id, q_filter=None):
|
||||
|
||||
try:
|
||||
qty, unit = METRICS_CONF['services_units'].get(
|
||||
resource_type,
|
||||
self.default_unit
|
||||
)
|
||||
tmp = METRICS_CONF['metrics_units'][resource_type]
|
||||
qty = list(tmp.keys())[0]
|
||||
unit = list(tmp.values())[0]['unit']
|
||||
# NOTE(mc): deprecated except part kept for backward compatibility.
|
||||
except KeyError:
|
||||
LOG.warning('Error when trying to use yaml metrology conf.')
|
||||
@ -311,7 +340,14 @@ class MonascaCollector(collector.BaseCollector):
|
||||
LOG.warning('Fallback on the deprecated oslo config method.')
|
||||
mappings = self.metrics_mappings[resource_type]
|
||||
|
||||
self._expand_metrics(data, resource_id, mappings, start, end)
|
||||
self._expand_metrics(
|
||||
data,
|
||||
resource_id,
|
||||
mappings,
|
||||
start,
|
||||
end,
|
||||
resource_type,
|
||||
)
|
||||
resource_qty = qty
|
||||
if not (isinstance(qty, int) or isinstance(qty, decimal.Decimal)):
|
||||
try:
|
||||
|
||||
@ -26,7 +26,7 @@ tests:
|
||||
$.services[/service_id][0].service_id: compute
|
||||
$.services[/service_id][0].unit: instance
|
||||
$.services[/service_id][1].service_id: image
|
||||
$.services[/service_id][1].unit: MB
|
||||
$.services[/service_id][1].unit: MiB
|
||||
$.services[/service_id][2].service_id: network.bw.in
|
||||
$.services[/service_id][2].unit: MB
|
||||
$.services[/service_id][3].service_id: network.bw.out
|
||||
@ -34,7 +34,7 @@ tests:
|
||||
$.services[/service_id][4].service_id: network.floating
|
||||
$.services[/service_id][4].unit: ip
|
||||
$.services[/service_id][5].service_id: volume
|
||||
$.services[/service_id][5].unit: GB
|
||||
$.services[/service_id][5].unit: GiB
|
||||
|
||||
- name: get compute service info
|
||||
url: /v1/info/services/compute
|
||||
|
||||
@ -24,6 +24,8 @@ to ease maintenance in case of library modifications.
|
||||
import calendar
|
||||
import contextlib
|
||||
import datetime
|
||||
import decimal
|
||||
import fractions
|
||||
import shutil
|
||||
import six
|
||||
import sys
|
||||
@ -275,3 +277,17 @@ def tempdir(**kwargs):
|
||||
except OSError as e:
|
||||
LOG.debug('Could not remove tmpdir: %s',
|
||||
six.text_type(e))
|
||||
|
||||
|
||||
def convert_unit(value, factor, offset=0):
|
||||
"""Return converted value depending on the provided factor and offset."""
|
||||
# Check if factor is a fraction
|
||||
if '/' in factor:
|
||||
tmp = fractions.Fraction(factor)
|
||||
numerator = decimal.Decimal(tmp.numerator)
|
||||
denominator = decimal.Decimal(tmp.denominator)
|
||||
factor = numerator / denominator
|
||||
else:
|
||||
factor = decimal.Decimal(factor)
|
||||
|
||||
return (decimal.Decimal(value) * factor) + decimal.Decimal(offset)
|
||||
|
||||
@ -188,6 +188,19 @@ The ``/etc/cloudkitty/metrics.yml`` file looks like this:
|
||||
.. literalinclude:: ../../../etc/cloudkitty/metrics.yml
|
||||
:language: yaml
|
||||
|
||||
Conversion information is included in the yaml file.
|
||||
It allows operators to change metrics units for rating
|
||||
and so to not stay stuck with the original unit.
|
||||
|
||||
The conversion information must be set for each metric.
|
||||
It includes optionals factor and offset,
|
||||
plus a mandatory final unit (used once the conversion is done).
|
||||
By default, factor and offset are 1 and 0 respectively.
|
||||
All type of linear conversions are so covered.
|
||||
The complete formula looks like:
|
||||
|
||||
``new_value = (value * factor) + offset``
|
||||
|
||||
|
||||
Setup the database and storage backend
|
||||
--------------------------------------
|
||||
|
||||
@ -44,20 +44,32 @@
|
||||
radosgw.usage:
|
||||
- radosgw.objects.size: max
|
||||
|
||||
services_units:
|
||||
metrics_units:
|
||||
compute:
|
||||
1: instance
|
||||
1:
|
||||
unit: instance
|
||||
volume:
|
||||
volume.size: GB
|
||||
volume.size:
|
||||
unit: GiB
|
||||
network.bw.in:
|
||||
network.incoming.bytes: MB
|
||||
network.incoming.bytes:
|
||||
unit: MB
|
||||
factor: 1/1000000
|
||||
network.bw.out:
|
||||
network.outgoing.bytes: MB
|
||||
network.outgoing.bytes:
|
||||
unit: MB
|
||||
factor: 1/1000000
|
||||
network.floating:
|
||||
1: ip
|
||||
1:
|
||||
unit: ip
|
||||
image:
|
||||
image.size: MB
|
||||
image.size:
|
||||
unit: MiB
|
||||
factor: 1/1048576
|
||||
radosgw.usage:
|
||||
radosgw.objects.size: GB
|
||||
radosgw.objects.size:
|
||||
unit: GiB
|
||||
factor: 1/1073741824
|
||||
default_unit:
|
||||
1: unknown
|
||||
1:
|
||||
unit: unknown
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user