From e605827a15101ddf5072d0654a8821220cc41891 Mon Sep 17 00:00:00 2001 From: Maxime Cottret Date: Tue, 17 Jan 2017 10:19:38 +0100 Subject: [PATCH] Add Ceph Object Storage Usage service This patch add the service "radosgw.usage" to collectors in order to allow rating on ceph object storage usage per project. The service unit is set to GB. The following work has been done: - Add radosgw.usage transformer for ceilometer - Add radosgw.usage collect function to ceilometer and gnocchi (new and deprecated format) - update doc Change-Id: Ia49c76a285950475ed0a9f15d35b2f23b11e5ef3 --- cloudkitty/collector/ceilometer.py | 25 +++++++++++++++++++++++++ cloudkitty/collector/gnocchi.py | 7 +++++++ cloudkitty/transformer/ceilometer.py | 9 +++++++++ doc/source/admin/rating/hashmap.rst | 2 ++ etc/cloudkitty/metrics.yml | 5 +++++ 5 files changed, 48 insertions(+) diff --git a/cloudkitty/collector/ceilometer.py b/cloudkitty/collector/ceilometer.py index 338db814..9fd9c19f 100644 --- a/cloudkitty/collector/ceilometer.py +++ b/cloudkitty/collector/ceilometer.py @@ -90,6 +90,7 @@ class CeilometerCollector(collector.BaseCollector): 'network.bw.out': 'MB', 'network.bw.in': 'MB', 'network.floating': 'ip', + 'radosgw.usage': 'GB' } def __init__(self, transformers, **kwargs): @@ -430,3 +431,27 @@ class CeilometerCollector(collector.BaseCollector): 'network.floating') return self.t_cloudkitty.format_service('network.floating', floating_data) + + def get_radosgw_usage(self, + start, + end=None, + project_id=None, + q_filter=None): + active_rgw_stats = self.resources_stats('radosgw.objects.size', start, + end, project_id, q_filter) + rgw_data = [] + for rgw_stats in active_rgw_stats: + rgw_id = rgw_stats.groupby['resource_id'] + if not self._cacher.has_resource_detail('radosgw.usage', rgw_id): + raw_resource = self._conn.resources.get(rgw_id) + rgw = self.t_ceilometer.strip_resource_data('radosgw.usage', + raw_resource) + self._cacher.add_resource_detail('radosgw.usage', rgw_id, rgw) + rgw = self._cacher.get_resource_detail('radosgw.usage', rgw_id) + rgw_size = decimal.Decimal(rgw_stats.max) / units.Gi + rgw_data.append(self.t_cloudkitty.format_item( + rgw, self.units_mappings["radosgw.usage"], rgw_size)) + if not rgw_data: + raise collector.NoDataCollected(self.collector_name, + 'radosgw.usage') + return self.t_cloudkitty.format_service('radosgw.usage', rgw_data) diff --git a/cloudkitty/collector/gnocchi.py b/cloudkitty/collector/gnocchi.py index 1a4551e4..51a086e7 100644 --- a/cloudkitty/collector/gnocchi.py +++ b/cloudkitty/collector/gnocchi.py @@ -53,6 +53,7 @@ class GnocchiCollector(collector.BaseCollector): 'network.bw.out': 'instance_network_interface', 'network.bw.in': 'instance_network_interface', 'network.floating': 'network', + 'radosgw.usage': 'ceph_account', } metrics_mappings = { 'compute': [ @@ -73,6 +74,8 @@ class GnocchiCollector(collector.BaseCollector): ('network.incoming.bytes', 'max')], 'network.floating': [ ('ip.floating', 'max')], + 'radosgw.usage': [ + ('radosgw.objects.size', 'max')], } units_mappings = { 'compute': (1, 'instance'), @@ -81,6 +84,7 @@ class GnocchiCollector(collector.BaseCollector): 'network.bw.out': ('network.outgoing.bytes', 'MB'), 'network.bw.in': ('network.incoming.bytes', 'MB'), 'network.floating': (1, 'ip'), + 'radosgw.usage': ('radosgw.objects.size', 'GB') } default_unit = (1, 'unknown') @@ -293,6 +297,9 @@ class GnocchiCollector(collector.BaseCollector): elif resource.get('type') == 'image': resource_data[qty] = ( decimal.Decimal(resource_data[qty]) / units.Mi) + elif resource.get('type') == 'ceph_account': + resource_data[qty] = ( + decimal.Decimal(resource_data[qty]) / units.Gi) data = self.t_cloudkitty.format_item( resource_data, unit, decimal.Decimal( diff --git a/cloudkitty/transformer/ceilometer.py b/cloudkitty/transformer/ceilometer.py index 90cb7617..683ba29f 100644 --- a/cloudkitty/transformer/ceilometer.py +++ b/cloudkitty/transformer/ceilometer.py @@ -65,6 +65,8 @@ class CeilometerTransformer(transformer.BaseTransformer): 'port_id': ['port_id'], } + radosgw_usage_map = {} + metadata_item = 'metadata' def _strip_compute(self, data): @@ -105,6 +107,13 @@ class CeilometerTransformer(transformer.BaseTransformer): res_data['floatingip_id'] = data.resource_id return res_data + def _strip_radosgw_usage(self, data): + res_data = self.generic_strip('radosgw_usage_size', data) + res_data['radosgw_id'] = data.resource_id + res_data['user_id'] = data.user_id + res_data['project_id'] = data.project_id + return res_data + def get_metadata(self, res_type): """Return list of metadata available after transformation for given diff --git a/doc/source/admin/rating/hashmap.rst b/doc/source/admin/rating/hashmap.rst index 2b794a84..38722bdb 100644 --- a/doc/source/admin/rating/hashmap.rst +++ b/doc/source/admin/rating/hashmap.rst @@ -34,6 +34,7 @@ the following services are available: * network.bw.in * network.bw.out * network.floating +* radosgw.usage Enabled services are defined in the configuration file. By default, only the compute service is enabled. @@ -97,6 +98,7 @@ For an active resource on a collection period, quantity is defined as follow: * network.bw.in: ingoing network usage (unit: MB) * network.bw.out: outgoing network usage (unit: MB) * network.floating: 1 (unit: ip) +* radosgw.usage: Ceph object storage usage (unit: GB) Example ======= diff --git a/etc/cloudkitty/metrics.yml b/etc/cloudkitty/metrics.yml index d43c975d..41a4155e 100644 --- a/etc/cloudkitty/metrics.yml +++ b/etc/cloudkitty/metrics.yml @@ -20,6 +20,7 @@ network.bw.in: instance_network_interface network.floating: network image: image + radosgw.usage: ceph_account services_metrics: compute: @@ -40,6 +41,8 @@ - image.size: max - image.download: max - image.serve: max + radosgw.usage: + - radosgw.objects.size: max services_units: compute: @@ -54,5 +57,7 @@ 1: ip image: image.size: MB + radosgw.usage: + radosgw.objects.size: GB default_unit: 1: unknown