openstack/cloudkitty
Code Issues Proposed changes

Allow Cloudkitty to collect non-OpenStack metrics

Browse Source 
* Add a SourceFetcher, which allows to retrieve new type of data sources.
  It allows new collectors to be added.
  (examples: Prometheus, Influxdb, Kubernetes...)

* Base Cloudkitty on metrics instead of services and resources
  for collecting metrics. This new architecture allows Cloudkitty
  to be more agnostic and so to rate containers metrics as the same way
  as virtual machines metrics.

* Centralize metrology information in metrics.yml under metrics names.

Task: 6291
Story: 2001501

Change-Id: I00ca080cf05dfc03a3363720f85b79e003eda9be
This commit is contained in:
Martin CAMEY 2018-03-07 11:53:28 +01:00
parent 4b9e916d3d
commit dff3e97b12
29 changed files with 785 additions and 570 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

99
cloudkitty/api/v1/controllers/info.py
View File

@ -16,6 +16,7 @@
# @author: Maxime Cottret
#
from oslo_config import cfg
from oslo_log import log as logging
import pecan
from pecan import rest
import six
@ -28,51 +29,99 @@ from cloudkitty import collector
from cloudkitty.common import policy
from cloudkitty import utils as ck_utils
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
METADATA = collector.get_collector_metadata()
def get_all_metrics():
METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
METADATA = collector.get_metrics_based_collector_metadata()
if 'metrics' not in METRICS_CONF:
msg = 'Invalid endpoint: no metrics in current configuration.'
pecan.abort(405, msg)
policy.authorize(pecan.request.context, 'info:list_metrics_info', {})
metrics_info_list = []
for metric, metadata in METADATA.items():
info = metadata.copy()
info['metric_id'] = metric
metrics_info_list.append(
info_models.CloudkittyMetricInfo(**info))
return info_models.CloudkittyMetricInfoCollection(
metrics=metrics_info_list)
def get_one_metric(metric_name):
METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
METADATA = collector.get_metrics_based_collector_metadata()
if 'metrics' not in METRICS_CONF:
msg = 'Invalid endpoint: no metrics in current configuration.'
pecan.abort(405, msg)
policy.authorize(pecan.request.context, 'info:get_metric_info', {})
try:
info = METADATA[metric_name].copy()
info['metric_id'] = metric_name
return info_models.CloudkittyMetricInfo(**info)
except KeyError:
pecan.abort(404, six.text_type(metric_name))
class MetricInfoController(rest.RestController):
"""REST Controller managing collected metrics information
independently of their services.
If no metrics are defined in conf, return 405 for each endpoint.
"""
@wsme_pecan.wsexpose(info_models.CloudkittyMetricInfoCollection)
def get_all(self):
"""Get the metric list.
:return: List of every metrics.
"""
return get_all_metrics()
@wsme_pecan.wsexpose(info_models.CloudkittyMetricInfo, wtypes.text)
def get_one(self, metric_name):
"""Return a metric.
:param metric_name: name of the metric.
"""
return get_one_metric(metric_name)
class ServiceInfoController(rest.RestController):
"""REST Controller mananging collected services information."""
"""REST Controller managing collected services information."""
@wsme_pecan.wsexpose(info_models.CloudkittyServiceInfoCollection)
@wsme_pecan.wsexpose(info_models.CloudkittyMetricInfoCollection)
def get_all(self):
"""Get the service list.
"""Get the service list (deprecated).
:return: List of every services.
"""
policy.authorize(pecan.request.context, 'info:list_services_info', {})
services_info_list = []
for service, metadata in METADATA.items():
info = metadata.copy()
info['service_id'] = service
services_info_list.append(
info_models.CloudkittyServiceInfo(**info))
return info_models.CloudkittyServiceInfoCollection(
services=services_info_list)
LOG.warning("Services based endpoints are deprecated. "
"Please use metrics based enpoints instead.")
return get_all_metrics()
@wsme_pecan.wsexpose(info_models.CloudkittyServiceInfo, wtypes.text)
@wsme_pecan.wsexpose(info_models.CloudkittyMetricInfo, wtypes.text)
def get_one(self, service_name):
"""Return a service.
"""Return a service (deprecated).
:param service_name: name of the service.
"""
policy.authorize(pecan.request.context, 'info:get_service_info', {})
try:
info = METADATA[service_name].copy()
info['service_id'] = service_name
return info_models.CloudkittyServiceInfo(**info)
except KeyError:
pecan.abort(404, six.text_type(service_name))
LOG.warning("Services based endpoints are deprecated. "
"Please use metrics based enpoints instead.")
return get_one_metric(service_name)
class InfoController(rest.RestController):
"""REST Controller managing Cloudkitty general information."""
services = ServiceInfoController()
metrics = MetricInfoController()
_custom_actions = {'config': ['GET']}
@ -82,6 +131,4 @@ class InfoController(rest.RestController):
def config(self):
"""Return current configuration."""
policy.authorize(pecan.request.context, 'info:get_config', {})
info = {}
info["collect"] = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
return info
return ck_utils.get_metrics_conf(CONF.collect.metrics_conf)

48
cloudkitty/api/v1/datamodels/info.py
View File

@ -18,33 +18,40 @@
from oslo_config import cfg
from wsme import types as wtypes
from cloudkitty.default_metrics_conf import DEFAULT_METRICS_CONF
from cloudkitty import utils as ck_utils
CONF = cfg.CONF
METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
CLOUDKITTY_SERVICES = wtypes.Enum(wtypes.text,
*METRICS_CONF['services'])
def get_metrics_list():
metrics_conf = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
try:
metrics = list(metrics_conf['metrics'].keys())
cloudkitty_metrics = wtypes.Enum(wtypes.text, *metrics)
except KeyError:
metrics = list(DEFAULT_METRICS_CONF['metrics'].keys())
cloudkitty_metrics = wtypes.Enum(wtypes.text, *metrics)
return cloudkitty_metrics
class CloudkittyServiceInfo(wtypes.Base):
"""Type describing a service info in CloudKitty.
class CloudkittyMetricInfo(wtypes.Base):
"""Type describing a metric info in CloudKitty."""
"""
service_id = CLOUDKITTY_SERVICES
"""Name of the service."""
metric_id = get_metrics_list()
"""Name of the metric."""
metadata = [wtypes.text]
"""List of service metadata"""
"""List of metric metadata"""
unit = wtypes.text
"""service unit"""
"""Metric unit"""
def to_json(self):
res_dict = {}
res_dict[self.service_id] = [{
res_dict[self.metric_id] = [{
'metadata': self.metadata,
'unit': self.unit
}]
@ -52,18 +59,19 @@ class CloudkittyServiceInfo(wtypes.Base):
@classmethod
def sample(cls):
sample = cls(service_id='compute',
metadata=['resource_id', 'flavor', 'availability_zone'],
unit='instance')
metadata = ['resource_id', 'project_id', 'qty', 'unit']
sample = cls(metric_id='image.size',
metadata=metadata,
unit='MiB')
return sample
class CloudkittyServiceInfoCollection(wtypes.Base):
"""A list of CloudKittyServiceInfo."""
class CloudkittyMetricInfoCollection(wtypes.Base):
"""A list of CloudKittyMetricInfo."""
services = [CloudkittyServiceInfo]
metrics = [CloudkittyMetricInfo]
@classmethod
def sample(cls):
sample = CloudkittyServiceInfo.sample()
return cls(services=[sample])
sample = CloudkittyMetricInfo.sample()
return cls(metrics=[sample])

17
cloudkitty/api/v1/datamodels/rating.py
View File

@ -27,8 +27,21 @@ CONF = cfg.CONF
METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
CLOUDKITTY_SERVICES = wtypes.Enum(wtypes.text,
*METRICS_CONF['services'])
try:
services_names = list(METRICS_CONF['services_objects'].keys())
except Exception:
# TODO(mc): remove this hack once rated dataframes are based on metrics.
services_names = [
'compute',
'volume',
'image',
'network.bw.in',
'network.bw.out',
'network.floating',
'radosgw.usage',
]
CLOUDKITTY_SERVICES = wtypes.Enum(wtypes.text, *services_names)
class CloudkittyResource(wtypes.Base):

45
cloudkitty/collector/__init__.py
View File

@ -27,37 +27,51 @@ from cloudkitty import utils as ck_utils
CONF = cfg.CONF
METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
COLLECTORS_NAMESPACE = 'cloudkitty.collector.backends'
def get_collector(transformers=None):
metrics_conf = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
if not transformers:
transformers = transformer.get_transformers()
collector_args = {
'period': METRICS_CONF['period'],
'transformers': transformers}
collector = driver.DriverManager(
'period': metrics_conf.get('period', 3600),
'transformers': transformers,
}
collector_args.update({'conf': metrics_conf})
return driver.DriverManager(
COLLECTORS_NAMESPACE,
METRICS_CONF['collector'],
metrics_conf.get('collector', 'gnocchi'),
invoke_on_load=True,
invoke_kwds=collector_args).driver
return collector
def get_collector_metadata():
def get_collector_without_invoke():
"""Return the collector without invoke it."""
metrics_conf = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
return driver.DriverManager(
COLLECTORS_NAMESPACE,
metrics_conf.get('collector', 'gnocchi'),
invoke_on_load=False
).driver
def get_metrics_based_collector_metadata():
"""Return dict of metadata.
Results are based on enabled collector and services in CONF.
Results are based on enabled collector and metrics in CONF.
"""
metrics_conf = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
transformers = transformer.get_transformers()
collector = driver.DriverManager(
COLLECTORS_NAMESPACE, METRICS_CONF['collector'],
invoke_on_load=False).driver
collector = get_collector_without_invoke()
metadata = {}
for service in METRICS_CONF['services']:
metadata[service] = collector.get_metadata(service, transformers)
if 'metrics' in metrics_conf:
for metric in metrics_conf.get('metrics', {}):
metadata[metric] = collector.get_metadata(
metric,
transformers,
metrics_conf,
)
return metadata
@ -94,6 +108,7 @@ class BaseCollector(object):
try:
self.transformers = transformers
self.period = kwargs['period']
self.conf = kwargs['conf']
except IndexError as e:
raise ValueError("Missing argument (%s)" % e)
@ -149,4 +164,4 @@ class BaseCollector(object):
"No method found in collector '%s' for resource '%s'."
% (self.collector_name, resource))
func = getattr(self, trans_resource)
return func(start, end, project_id, q_filter)
return func(resource, start, end, project_id, q_filter)

159
cloudkitty/collector/gnocchi.py
View File

@ -19,7 +19,6 @@ from gnocchiclient import client as gclient
from keystoneauth1 import loading as ks_loading
from oslo_config import cfg
from oslo_log import log as logging
from oslo_utils import units
from cloudkitty import collector
from cloudkitty import utils as ck_utils
@ -43,8 +42,6 @@ ks_loading.register_auth_conf_options(
GNOCCHI_COLLECTOR_OPTS)
CONF = cfg.CONF
METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
class GnocchiCollector(collector.BaseCollector):
collector_name = 'gnocchi'
@ -112,22 +109,13 @@ class GnocchiCollector(collector.BaseCollector):
'interface': CONF.gnocchi_collector.interface})
@classmethod
def get_metadata(cls, resource_name, transformers):
def get_metadata(cls, resource_name, transformers, conf):
info = super(GnocchiCollector, cls).get_metadata(resource_name,
transformers)
try:
info["metadata"].extend(transformers['GnocchiTransformer']
.get_metadata(resource_name))
try:
tmp = METRICS_CONF['metrics_units'][resource_name]
info['unit'] = list(tmp.values())[0]['unit']
# NOTE(mc): deprecated except part kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
info['unit'] = cls.units_mappings[resource_name][1]
info['unit'] = conf['metrics'][resource_name]['unit']
except KeyError:
pass
return info
@ -192,38 +180,20 @@ class GnocchiCollector(collector.BaseCollector):
# [point_date, granularity, value]
# ["2015-11-24T00:00:00+00:00", 86400.0, 64.0]
resource[name] = values[0][2]
except IndexError:
except (IndexError, KeyError):
resource[name] = 0
except KeyError:
# Skip metrics not found
pass
def _expand_metrics(self, resources, mappings, start, end):
def _expand_metrics(self, resources, mappings, start, end, resource_name):
for resource in resources:
metrics = resource.get('metrics', {})
try:
for mapping in mappings:
self._expand(
metrics,
resource,
mapping.keys()[0],
mapping.values()[0],
start,
end,
)
# NOTE(mc): deprecated except part kept for backward compatibility.
except AttributeError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
for name, aggregate in mappings:
self._expand(
metrics,
resource,
name,
aggregate,
start,
end,
)
self._expand(
metrics,
resource,
resource_name,
mappings,
start,
end,
)
def get_resources(self, resource_name, start, end,
project_id, q_filter=None):
@ -246,13 +216,7 @@ class GnocchiCollector(collector.BaseCollector):
# Translating the resource name if needed
query_parameters = self._generate_time_filter(start, end)
try:
resource_type = METRICS_CONF['services_objects'].get(resource_name)
# NOTE(mc): deprecated except part kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
resource_type = self.retrieve_mappings.get(resource_name)
resource_type = self.conf['metrics'][resource_name]['resource']
query_parameters.append(
self.gen_filter(cop="=", type=resource_type))
@ -265,72 +229,50 @@ class GnocchiCollector(collector.BaseCollector):
query=self.extend_filter(*query_parameters))
return resources
def resource_info(self, resource_name, start, end, project_id,
q_filter=None):
try:
tmp = METRICS_CONF['metrics_units'][resource_name]
qty = list(tmp.keys())[0]
unit = list(tmp.values())[0]['unit']
# NOTE(mc): deprecated except part kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
qty, unit = self.units_mappings.get(
resource_name,
self.default_unit,
)
def resource_info(self, resource_name, start, end,
project_id, q_filter=None):
met = self.conf['metrics'][resource_name]
unit = met['unit']
qty = 1 if met.get('countable_unit') else met['resource']
resources = self.get_resources(
resource_name,
start,
end,
project_id=project_id,
q_filter=q_filter,
)
resources = self.get_resources(resource_name, start, end,
project_id=project_id,
q_filter=q_filter)
formated_resources = list()
for resource in resources:
resource_data = self.t_gnocchi.strip_resource_data(
resource_name, resource)
try:
mappings = METRICS_CONF['services_metrics'][resource_name]
# NOTE(mc): deprecated except part kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
mappings = self.metrics_mappings[resource_name]
mapp = self.conf['metrics'][resource_name]['aggregation_method']
self._expand_metrics(
[resource_data],
mapp,
start,
end,
resource_name,
)
self._expand_metrics([resource_data], mappings, start, end)
resource_data.pop('metrics', None)
# Unit conversion
try:
conv_data = METRICS_CONF['metrics_units'][resource_name][qty]
if isinstance(qty, str):
resource_data[qty] = ck_utils.convert_unit(
resource_data[qty],
conv_data.get('factor', '1'),
conv_data.get('offset', '0'),
)
# NOTE(mc): deprecated except part kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated hardcoded method.')
if resource.get('type') == 'instance_network_interface':
resource_data[qty] = (
decimal.Decimal(resource_data[qty]) / units.M
)
elif resource.get('type') == 'image':
resource_data[qty] = (
decimal.Decimal(resource_data[qty]) / units.Mi
)
elif resource.get('type') == 'ceph_account':
resource_data[qty] = (
decimal.Decimal(resource_data[qty]) / units.Gi)
if isinstance(qty, str):
resource_data[resource_name] = ck_utils.convert_unit(
resource_data[resource_name],
self.conf['metrics'][resource_name].get('factor', 1),
self.conf['metrics'][resource_name].get('offset', 0),
)
val = qty if isinstance(qty, int) else resource_data[resource_name]
data = self.t_cloudkitty.format_item(
resource_data,
unit,
decimal.Decimal(
qty if isinstance(qty, int) else resource_data[qty]
)
decimal.Decimal(val)
)
# NOTE(sheeprine): Reference to gnocchi resource used by storage
@ -338,10 +280,17 @@ class GnocchiCollector(collector.BaseCollector):
formated_resources.append(data)
return formated_resources
def retrieve(self, resource_name, start, end, project_id, q_filter=None):
resources = self.resource_info(resource_name, start, end,
project_id=project_id,
q_filter=q_filter)
def retrieve(self, resource_name, start, end,
project_id, q_filter=None):
resources = self.resource_info(
resource_name,
start,
end,
project_id,
q_filter=q_filter,
)
if not resources:
raise collector.NoDataCollected(self.collector_name, resource_name)
return self.t_cloudkitty.format_service(resource_name, resources)

143
cloudkitty/collector/monasca.py
View File

@ -22,7 +22,6 @@ from keystoneclient.v3 import client as ks_client
from monascaclient import client as mclient
from oslo_config import cfg
from oslo_log import log as logging
from oslo_utils import units
from cloudkitty import collector
from cloudkitty import transformer
@ -130,55 +129,59 @@ class MonascaCollector(collector.BaseCollector):
return endpoint.url
return None
def _get_metadata(self, resource_type, transformers):
def _get_metadata(self, resource_type, transformers, conf):
info = {}
try:
met = list(METRICS_CONF['metrics_units'][resource_type].values())
info['unit'] = met[0]['unit']
# NOTE(mc): deprecated second try kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
try:
info['unit'] = self.units_mappings[resource_type][1]
except (KeyError, IndexError):
info['unit'] = self.default_unit[1]
info['unit'] = conf['metrics'][resource_type]['unit']
start = ck_utils.dt2ts(ck_utils.get_month_start())
end = ck_utils.dt2ts(ck_utils.get_month_end())
try:
resource = self.active_resources(resource_type, start,
end, None)[0]
resource = self.active_resources(
resource_type,
start,
end,
None,
)[0]
except IndexError:
resource = {}
info['metadata'] = resource.get('dimensions', {}).keys()
try:
service_metrics = METRICS_CONF['services_metrics'][resource_type]
for service_metric in service_metrics:
metric, statistics = list(service_metric.items())[0]
info['metadata'].append(metric)
# NOTE(mc): deprecated second try kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
try:
for metric, statistics in self.metrics_mappings[resource_type]:
info['metadata'].append(metric)
except (KeyError, IndexError):
pass
service_metrics = METRICS_CONF['services_metrics'][resource_type]
for service_metric in service_metrics:
metric, statistics = list(service_metric.items())[0]
info['metadata'].append(metric)
return info
# NOTE(lukapeschke) if anyone sees a better way to do this,
# please make a patch
@classmethod
def get_metadata(cls, resource_type, transformers):
def get_metadata(cls, resource_type, transformers, conf):
args = {
'transformers': transformer.get_transformers(),
'period': CONF.collect.period}
'period': conf['period']}
tmp = cls(**args)
return tmp._get_metadata(resource_type, transformers)
return tmp._get_metadata(resource_type, transformers, conf)
def _get_resource_metadata(self, resource_type, start,
end, resource_id, conf):
meter = conf['metrics'][resource_type]['resource']
if not meter:
return {}
measurements = self._conn.metrics.list_measurements(
name=meter,
start_time=ck_utils.ts2dt(start),
end_time=ck_utils.ts2dt(end),
merge_metrics=True,
dimensions={'resource_id': resource_id},
)
try:
# Getting the last measurement of given period
metadata = measurements[-1]['measurements'][-1][2]
except (KeyError, IndexError):
metadata = {}
return metadata
def _get_resource_qty(self, meter, start, end, resource_id, statistics):
# NOTE(lukapeschke) the period trick is used to aggregate
@ -213,14 +216,8 @@ class MonascaCollector(collector.BaseCollector):
return len(measurements) > 0
def active_resources(self, resource_type, start,
end, project_id, **kwargs):
try:
meter = METRICS_CONF['services_objects'].get(resource_type)
# NOTE(mc): deprecated except part kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
meter = self.retrieve_mappings.get(resource_type)
end, project_id, conf, **kwargs):
meter = conf['metrics'][resource_type]['resource']
if not meter:
return {}
@ -254,58 +251,32 @@ class MonascaCollector(collector.BaseCollector):
statistics,
)
try:
conv_data = METRICS_CONF['metrics_units'][resource_type]
conv_data = conv_data.get(name)
if conv_data:
resource[name] = ck_utils.convert_unit(
qty,
conv_data.get('factor', 1),
conv_data.get('offset', 0),
)
# NOTE(mc): deprecated except part kept for backward compatibility.
except KeyError:
LOG.warning(
'Error when trying to use yaml metrology conf.\n'
'Fallback on the deprecated hardcoded dict method.')
names = ['network.outgoing.bytes', 'network.incoming.bytes']
if name in names:
qty = qty / units.M
elif 'image.' in name:
qty = qty / units.Mi
resource[name] = qty
conv_data = METRICS_CONF['metrics'][resource_type].get(name)
if conv_data:
resource[name] = ck_utils.convert_unit(
qty,
conv_data.get('factor', 1),
conv_data.get('offset', 0),
)
def resource_info(self, resource_type, start, end,
project_id, q_filter=None):
try:
tmp = METRICS_CONF['metrics_units'][resource_type]
qty = list(tmp.keys())[0]
unit = list(tmp.values())[0]['unit']
# NOTE(mc): deprecated except part kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
qty, unit = self.units_mappings.get(
resource_type,
self.default_unit
)
met = self.conf['metrics'][resource_type]
unit = met['unit']
qty = 1 if met.get('countable_unit') else met['resource']
active_resources = self.active_resources(
resource_type, start, end, project_id
)
resource_data = []
for resource in active_resources:
resource_id = resource['dimensions']['resource_id']
data = resource['dimensions']
try:
mappings = METRICS_CONF['services_metrics'][resource_type]
# NOTE(mc): deprecated except part kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
mappings = self.metrics_mappings[resource_type]
mappings = (
resource_type,
METRICS_CONF['metrics'][resource_type]['aggregation_method'],
)
self._expand_metrics(
data,
@ -317,15 +288,7 @@ class MonascaCollector(collector.BaseCollector):
)
resource_qty = qty
if not (isinstance(qty, int) or isinstance(qty, decimal.Decimal)):
try:
resource_qty \
= METRICS_CONF['services_objects'][resource_type]
# NOTE(mc): deprecated except part kept for backward compat.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf')
msg = 'Fallback on the deprecated oslo config method'
LOG.warning(msg)
resource_qty = data[self.retrieve_mappings[resource_type]]
resource_qty = METRICS_CONF['services_objects'][resource_type]
resource_qty = data[resource_qty]
resource = self.t_cloudkitty.format_item(data, unit, resource_qty)

16
cloudkitty/common/config.py
View File

@ -20,12 +20,12 @@ import cloudkitty.api.app
import cloudkitty.collector.gnocchi
import cloudkitty.collector.monasca
import cloudkitty.config
import cloudkitty.fetcher
import cloudkitty.fetcher.keystone
import cloudkitty.orchestrator
import cloudkitty.service
import cloudkitty.storage
import cloudkitty.storage.hybrid.backends.gnocchi
import cloudkitty.tenant_fetcher
import cloudkitty.tenant_fetcher.keystone
import cloudkitty.utils
__all__ = ['list_opts']
@ -41,8 +41,8 @@ _opts = [
cloudkitty.collector.gnocchi.end_point_type_opts,
cloudkitty.collector.gnocchi.gnocchi_collector_opts))),
('keystone_fetcher', list(itertools.chain(
cloudkitty.tenant_fetcher.keystone.keystone_fetcher_opts,
cloudkitty.tenant_fetcher.keystone.keystone_common_opts))),
cloudkitty.fetcher.keystone.keystone_fetcher_opts,
cloudkitty.fetcher.keystone.keystone_common_opts))),
('orchestrator', list(itertools.chain(
cloudkitty.orchestrator.orchestrator_opts))),
('output', list(itertools.chain(
@ -50,14 +50,14 @@ _opts = [
('state', list(itertools.chain(
cloudkitty.config.state_opts))),
('storage', list(itertools.chain(
cloudkitty.storage.storage_opts))),
cloudkitty.utils.storage_opts))),
('storage_gnocchi', list(itertools.chain(
cloudkitty.storage.hybrid.backends.gnocchi.gnocchi_storage_opts))),
('tenant_fetcher', list(itertools.chain(
cloudkitty.tenant_fetcher.fetchers_opts))),
('fetcher', list(itertools.chain(
cloudkitty.fetcher.fetchers_opts))),
(None, list(itertools.chain(
cloudkitty.api.app.auth_opts,
cloudkitty.service.service_opts)))
cloudkitty.service.service_opts))),
]

14
cloudkitty/common/policies/info.py
View File

@ -28,7 +28,19 @@ info_policies = [
name='info:get_service_info',
check_str=base.UNPROTECTED,
description='Get specified service information.',
operations=[{'path': '/v1/info/services/{service_id}',
operations=[{'path': '/v1/info/services/{metric_id}',
'method': 'GET'}]),
policy.DocumentedRuleDefault(
name='info:list_metrics_info',
check_str=base.UNPROTECTED,
description='List available metrics information in Cloudkitty.',
operations=[{'path': '/v1/info/metrics',
'method': 'LIST'}]),
policy.DocumentedRuleDefault(
name='info:get_metric_info',
check_str=base.UNPROTECTED,
description='Get specified metric information.',
operations=[{'path': '/v1/info/metrics/{metric_id}',
'method': 'GET'}]),
policy.DocumentedRuleDefault(
name='info:get_config',

124
cloudkitty/default_metrics_conf.py Normal file
View File

@ -0,0 +1,124 @@
# -*- coding: utf-8 -*-
# Copyright 2018 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# @author: Martin CAMEY
#
DEFAULT_METRICS_CONF = {
'name': 'OpenStack',
'fetcher': 'keystone',
'collector': 'gnocchi',
'period': 3600,
'wait_periods': 2,
'window': 1800,
'services_objects': {
'compute': 'instance',
'volume': 'volume',
'network.bw.out': 'instance_network_interface',
'network.bw.in': 'instance_network_interface',
'network.floating': 'network',
'image': 'image',
'radosgw.usage': 'ceph_account',
},
'metrics': {
'vcpus': {
'resource': 'instance',
'unit': 'instance',
'factor': 1,
'aggregation_method': 'max',
'countable_unit': True,
},
'memory': {
'resource': 'instance',
'unit': 'instance',
'factor': 1,
'aggregation_method': 'max',
'countable_unit': True,
},
'cpu': {
'resource': 'instance',
'unit': 'instance',
'factor': 1,
'aggregation_method': 'max',
'countable_unit': True,
},
'disk.root.size': {
'resource': 'instance',
'unit': 'instance',
'factor': 1,
'aggregation_method': 'max',
'countable_unit': True,
},
'disk.ephemeral.size': {
'resource': 'instance',
'unit': 'instance',
'factor': 1,
'aggregation_method': 'max',
'countable_unit': True,
},
'image.size': {
'resource': 'image',
'unit': 'MiB',
'factor': 1 / 1048576,
'aggregation_method': 'max',
},
'image.download': {
'resource': 'image',
'unit': 'MiB',
'factor': 1 / 1048576,
'aggregation_method': 'max',
},
'image.serve': {
'resource': 'image',
'unit': 'MiB',
'factor': 1 / 1048576,
'aggregation_method': 'max',
},
'volume.size': {
'resource': 'volume',
'unit': 'GiB',
'factor': 1,
'aggregation_method': 'max',
},
'network.outgoing.bytes': {
'resource': 'instance_network_interface',
'unit': 'MB',
'factor': 1 / 1000000,
'aggregation_method': 'max',
},
'network.incoming.bytes': {
'resource': 'instance_network_interface',
'unit': 'MB',
'factor': 1 / 1000000,
'aggregation_method': 'max',
},
'ip.floating': {
'resource': 'network',
'unit': 'ip',
'factor': 1,
'aggregation_method': 'max',
'countable_unit': True,
},
'radosgw.objects.size': {
'resource': 'ceph_account',
'unit': 'GiB',
'factor': 1 / 1073741824,
'aggregation_method': 'max',
},
},
}

0
cloudkitty/tenant_fetcher/__init__.py → cloudkitty/fetcher/__init__.py
View File

4
cloudkitty/tenant_fetcher/fake.py → cloudkitty/fetcher/fake.py
View File

@ -20,7 +20,7 @@ import csv
from oslo_config import cfg
from cloudkitty import tenant_fetcher
from cloudkitty import fetcher
fake_fetcher_opts = [
cfg.StrOpt('file',
@ -31,7 +31,7 @@ cfg.CONF.register_opts(fake_fetcher_opts, 'fake_fetcher')
CONF = cfg.CONF
class FakeFetcher(tenant_fetcher.BaseFetcher):
class FakeFetcher(fetcher.BaseFetcher):
"""Fake tenants fetcher."""
def __init__(self):

19
cloudkitty/tenant_fetcher/keystone.py → cloudkitty/fetcher/keystone.py
View File

@ -22,7 +22,8 @@ from keystoneclient import discover
from keystoneclient import exceptions
from oslo_config import cfg
from cloudkitty import tenant_fetcher
from cloudkitty import fetcher
KEYSTONE_FETCHER_OPTS = 'keystone_fetcher'
keystone_common_opts = ks_loading.get_auth_common_conf_options()
@ -42,9 +43,11 @@ ks_loading.register_auth_conf_options(
CONF = cfg.CONF
class KeystoneFetcher(tenant_fetcher.BaseFetcher):
class KeystoneFetcher(fetcher.BaseFetcher):
"""Keystone tenants fetcher."""
name = 'keystone'
def __init__(self):
self.auth = ks_loading.load_auth_from_conf_options(
CONF,
@ -58,18 +61,18 @@ class KeystoneFetcher(tenant_fetcher.BaseFetcher):
session=self.session,
auth_url=self.auth.auth_url)
def get_tenants(self):
def get_tenants(self, conf=None):
keystone_version = discover.normalize_version_number(
CONF.keystone_fetcher.keystone_version)
auth_dispatch = {(3,): ('project', 'projects', 'list'),
(2,): ('tenant', 'tenants', 'roles_for_user')}
for auth_version, auth_version_mapping in auth_dispatch.items():
if discover.version_match(auth_version, keystone_version):
return self._do_get_tenants(auth_version_mapping)
return self._do_get_tenants(auth_version_mapping, conf)
msg = "Keystone version you've specified is not supported"
raise exceptions.VersionNotAvailable(msg)
def _do_get_tenants(self, auth_version_mapping):
def _do_get_tenants(self, auth_version_mapping, conf):
tenant_attr, tenants_attr, role_func = auth_version_mapping
tenant_list = getattr(self.admin_ks, tenants_attr).list()
my_user_id = self.session.get_user_id()
@ -79,4 +82,8 @@ class KeystoneFetcher(tenant_fetcher.BaseFetcher):
tenant_attr: tenant})
if 'rating' not in [role.name for role in roles]:
tenant_list.remove(tenant)
return [tenant.id for tenant in tenant_list]
if conf:
res = [{'tenant_id': tenant.id} for tenant in tenant_list]
for tenant in res:
tenant.update(conf)
return res

37
cloudkitty/fetcher/source.py Normal file
View File

@ -0,0 +1,37 @@
# -*- coding: utf-8 -*-
# !/usr/bin/env python
# Copyright 2015 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# @author: Martin CAMEY
#
import hashlib
from cloudkitty import fetcher
class SourceFetcher(fetcher.BaseFetcher):
"""Source projects fetcher."""
name = 'source'
def get_projects(self, conf=None):
if conf:
tmp = hashlib.md5()
tmp.update(conf['name'])
conf['tenant_id'] = tmp.hexdigest()
return [conf]
def get_tenants(self, conf=None):
return self.get_projects(conf=conf)

109
cloudkitty/orchestrator.py
View File

@ -41,7 +41,7 @@ eventlet.monkey_patch()
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
CONF.import_opt('backend', 'cloudkitty.tenant_fetcher', 'tenant_fetcher')
CONF.import_opt('backend', 'cloudkitty.fetcher', 'tenant_fetcher')
orchestrator_opts = [
cfg.StrOpt('coordination_url',
@ -55,6 +55,8 @@ METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
FETCHERS_NAMESPACE = 'cloudkitty.tenant.fetchers'
PROCESSORS_NAMESPACE = 'cloudkitty.rating.processors'
COLLECTORS_NAMESPACE = 'cloudkitty.collector.backends'
STORAGES_NAMESPACE = 'cloudkitty.storage.backends'
class RatingEndpoint(object):
@ -151,20 +153,26 @@ class APIWorker(BaseWorker):
class Worker(BaseWorker):
def __init__(self, collector, storage, tenant_id=None):
def __init__(self, collector, storage, tenant):
self._collector = collector
self._storage = storage
self._period = METRICS_CONF['period']
self._wait_time = METRICS_CONF['wait_periods'] * self._period
self._period = tenant['period']
self._wait_time = tenant['wait_periods'] * self._period
self._tenant_id = tenant['tenant_id']
self.conf = tenant
super(Worker, self).__init__(tenant_id)
super(Worker, self).__init__(self._tenant_id)
def _collect(self, service, start_timestamp):
def _collect(self, metric, start_timestamp):
next_timestamp = start_timestamp + self._period
raw_data = self._collector.retrieve(service,
start_timestamp,
next_timestamp,
self._tenant_id)
raw_data = self._collector.retrieve(
metric,
start_timestamp,
next_timestamp,
self._tenant_id,
)
if raw_data:
return [{'period': {'begin': start_timestamp,
'end': next_timestamp},
@ -182,18 +190,20 @@ class Worker(BaseWorker):
if not timestamp:
break
for service in METRICS_CONF['services']:
metrics = list(self.conf['metrics'].keys())
for metric in metrics:
try:
try:
data = self._collect(service, timestamp)
data = self._collect(metric, timestamp)
except collector.NoDataCollected:
raise
except Exception as e:
LOG.warning(
'Error while collecting service '
'%(service)s: %(error)s',
{'service': service, 'error': e})
raise collector.NoDataCollected('', service)
'Error while collecting metric '
'%(metric)s: %(error)s',
{'metric': metric, 'error': e})
raise collector.NoDataCollected('', metric)
except collector.NoDataCollected:
begin = timestamp
end = begin + self._period
@ -213,15 +223,15 @@ class Worker(BaseWorker):
class Orchestrator(object):
def __init__(self):
# Tenant fetcher
self.fetcher = driver.DriverManager(
FETCHERS_NAMESPACE,
CONF.tenant_fetcher.backend,
invoke_on_load=True).driver
METRICS_CONF['fetcher'],
invoke_on_load=True
).driver
self.transformers = transformer.get_transformers()
self.collector = collector.get_collector(self.transformers)
self.storage = storage.get_storage(self.collector)
transformers = transformer.get_transformers()
self.collector = collector.get_collector(transformers)
self.storage = storage.get_storage(collector=self.collector)
# RPC
self.server = None
@ -234,17 +244,10 @@ class Orchestrator(object):
uuidutils.generate_uuid().encode('ascii'))
self.coord.start()
self._period = METRICS_CONF['period']
self._wait_time = METRICS_CONF['wait_periods'] * self._period
def _lock(self, tenant_id):
lock_name = b"cloudkitty-" + str(tenant_id).encode('ascii')
return self.coord.get_lock(lock_name)
def _load_tenant_list(self):
self._tenants = self.fetcher.get_tenants()
random.shuffle(self._tenants)
def _init_messaging(self):
target = oslo_messaging.Target(topic='cloudkitty',
server=CONF.host,
@ -255,11 +258,11 @@ class Orchestrator(object):
self.server = messaging.get_server(target, endpoints)
self.server.start()
def _check_state(self, tenant_id):
def _check_state(self, tenant_id, period, wait_time):
timestamp = self.storage.get_state(tenant_id)
return ck_utils.check_time_state(timestamp,
self._period,
self._wait_time)
period,
wait_time)
def process_messages(self):
# TODO(sheeprine): Code kept to handle threading and asynchronous
@ -270,26 +273,36 @@ class Orchestrator(object):
def process(self):
while True:
self.process_messages()
self._load_tenant_list()
while len(self._tenants):
for tenant in self._tenants[:]:
lock = self._lock(tenant)
if lock.acquire(blocking=False):
if not self._check_state(tenant):
self._tenants.remove(tenant)
else:
worker = Worker(self.collector,
self.storage,
tenant)
worker.run()
lock.release()
self.coord.heartbeat()
self.tenants = self.fetcher.get_tenants(METRICS_CONF)
random.shuffle(self.tenants)
LOG.info('Tenants loaded for fetcher %s', self.fetcher.name)
for tenant in self.tenants:
lock = self._lock(tenant['tenant_id'])
if lock.acquire(blocking=False):
state = self._check_state(
tenant['tenant_id'],
tenant['period'],
tenant['wait_periods'],
)
if not state:
self.tenants.remove(tenant)
else:
worker = Worker(
self.collector,
self.storage,
tenant,
)
worker.run()
lock.release()
self.coord.heartbeat()
# NOTE(sheeprine): Slow down looping if all tenants are
# being processed
eventlet.sleep(1)
# FIXME(sheeprine): We may cause a drift here
eventlet.sleep(self._period)
eventlet.sleep(tenant['period'])
def terminate(self):
self.coord.stop()

22
cloudkitty/storage/__init__.py
View File

@ -25,31 +25,31 @@ from stevedore import driver
from cloudkitty import collector as ck_collector
from cloudkitty import utils as ck_utils
LOG = logging.getLogger(__name__)
storage_opts = [
cfg.StrOpt('backend',
default='sqlalchemy',
help='Name of the storage backend driver.')
]
CONF = cfg.CONF
CONF.register_opts(storage_opts, group='storage')
# NOTE(mc): This hack is possible because only
# one OpenStack configuration is allowed.
METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
STORAGES_NAMESPACE = 'cloudkitty.storage.backends'
def get_storage(collector=None):
if not collector:
collector = ck_collector.get_collector()
storage_args = {
'period': METRICS_CONF['period'],
'collector': collector if collector else ck_collector.get_collector()}
'period': METRICS_CONF.get('period', 3600),
'collector': collector,
}
backend = driver.DriverManager(
STORAGES_NAMESPACE,
cfg.CONF.storage.backend,
invoke_on_load=True,
invoke_kwds=storage_args).driver
invoke_kwds=storage_args
).driver
return backend
@ -68,7 +68,7 @@ class BaseStorage(object):
Handle incoming data from the global orchestrator, and store them.
"""
def __init__(self, **kwargs):
self._period = kwargs.get('period', METRICS_CONF['period'])
self._period = kwargs.get('period')
self._collector = kwargs.get('collector')
# State vars

50
cloudkitty/storage/hybrid/backends/gnocchi.py
View File

@ -36,6 +36,10 @@ LOG = logging.getLogger(__name__)
CONF = cfg.CONF
# NOTE(mc): This hack is possible because only
# one OpenStack configuration is allowed.
METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
CONF.import_opt('period', 'cloudkitty.collector', 'collect')
GNOCCHI_STORAGE_OPTS = 'storage_gnocchi'
@ -49,7 +53,7 @@ gnocchi_storage_opts = [
# The archive policy definition MUST include the collect period granularity
cfg.StrOpt('archive_policy_definition',
default='[{"granularity": '
+ six.text_type(CONF.collect.period) +
+ six.text_type(METRICS_CONF.get('period', 3600)) +
', "timespan": "90 days"}, '
'{"granularity": 86400, "timespan": "360 days"}, '
'{"granularity": 2592000, "timespan": "1800 days"}]',
@ -63,8 +67,6 @@ ks_loading.register_auth_conf_options(
CONF,
GNOCCHI_STORAGE_OPTS)
METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
RESOURCE_TYPE_NAME_ROOT = 'rating_service_'
@ -100,37 +102,24 @@ class GnocchiStorage(BaseHybridBackend):
"creator",
]
@staticmethod
def _get_service_metrics(service_name):
metrics = METRICS_CONF['services_metrics'][service_name]
metric_list = ['price']
for metric in metrics:
metric_list.append(list(metric.keys())[0])
return metric_list
def _init_resource_types(self):
transformer = gtransformer.GnocchiTransformer()
services = METRICS_CONF['services']
for service in services:
service_dict = dict()
service_dict['attributes'] = list()
for attribute in transformer.get_metadata(service):
for metric in list(self.conf['metrics'].keys()):
metric_dict = dict()
metric_dict['attributes'] = list()
for attribute in transformer.get_metadata(metric):
if attribute not in self.invalid_attribute_names:
service_dict['attributes'].append(attribute)
service_dict['required_attributes'] = [
metric_dict['attributes'].append(attribute)
metric_dict['required_attributes'] = [
'resource_id',
'unit',
]
try:
service_dict['metrics'] = self._get_service_metrics(service)
except KeyError:
LOG.warning(
'No metrics configured for service {}'.format(service))
service_dict['metrics'] = list()
service_dict['name'] = RESOURCE_TYPE_NAME_ROOT + service
service_dict['qty_metric'] \
= list(METRICS_CONF['metrics_units'][service].keys())[0]
self._resource_type_data[service] = service_dict
metric_dict['name'] = RESOURCE_TYPE_NAME_ROOT + metric
metric_dict['qty_metric'] = 1
if self.conf['metrics'][metric].get('countable_unit'):
resource = self.conf['metrics'][metric]['resource']
metric_dict['qty_metric'] = resource
self._resource_type_data[metric] = metric_dict
def _get_res_type_dict(self, res_type):
res_type_data = self._resource_type_data.get(res_type, None)
@ -175,7 +164,7 @@ class GnocchiStorage(BaseHybridBackend):
'name': metric,
'archive_policy_name':
CONF.storage_gnocchi.archive_policy_name,
}) for metric in res_type_data['metrics']
}) for metric in ['price', res_type]
]
metrics_dict = dict()
@ -235,6 +224,7 @@ class GnocchiStorage(BaseHybridBackend):
def __init__(self, **kwargs):
super(GnocchiStorage, self).__init__(**kwargs)
self.conf = kwargs['conf'] if 'conf' in kwargs else METRICS_CONF
self.auth = ks_loading.load_auth_from_conf_options(
CONF,
GNOCCHI_STORAGE_OPTS)
@ -251,7 +241,7 @@ class GnocchiStorage(BaseHybridBackend):
CONF.storage_gnocchi.archive_policy_name)
self._archive_policy_definition = json.loads(
CONF.storage_gnocchi.archive_policy_definition)
self._period = CONF.collect.period
self._period = self.conf['period']
if "period" in kwargs:
self._period = kwargs["period"]
self._measurements = dict()

14
cloudkitty/tests/gabbi/fixtures.py
View File

@ -387,6 +387,20 @@ class CORSConfigFixture(fixture.GabbiFixture):
cfg.ConfigOpts.GroupAttr.__getattr__ = self._original_call_method
class MetricsConfFixture(fixture.GabbiFixture):
"""Inject Metrics configuration mock to the get_metrics_conf() function"""
def start_fixture(self):
self._original_function = ck_utils.get_metrics_conf
ck_utils.get_metrics_conf = mock.Mock(
return_value=tests.samples.METRICS_CONF,
)
def stop_fixture(self):
"""Remove the get_metrics_conf() monkeypatch."""
ck_utils.get_metrics_conf = self._original_function
def setup_app():
messaging.setup()
# FIXME(sheeprine): Extension fixtures are interacting with transformers

83
cloudkitty/tests/gabbi/gabbits/v1-info.yaml
View File

@ -1,45 +1,68 @@
fixtures:
- ConfigFixture
- MetricsConfFixture
tests:
- name: get config
url: /v1/info/config
status: 200
response_json_paths:
$.collect.services.`len`: 6
$.collect.services[0]: compute
$.collect.services[1]: volume
$.collect.services[2]: network.bw.in
$.collect.services[3]: network.bw.out
$.collect.services[4]: network.floating
$.collect.services[5]: image
$.collect.collector: gnocchi
$.collect.window: 1800
$.collect.wait_periods: 2
$.collect.period: 3600
$.collector: gnocchi
$.window: 1800
$.wait_periods: 2
$.period: 3600
- name: get services info
url: /v1/info/services
$.metrics.`len`: 13
$.metrics.vcpus.unit: instance
$.metrics.memory.unit: instance
$.metrics.cpu.unit: instance
$.metrics['disk.root.size'].unit: instance
$.metrics['disk.ephemeral.size'].unit: instance
$.metrics['image.size'].unit: MiB
$.metrics['image.download'].unit: MiB
$.metrics['image.serve'].unit: MiB
$.metrics['volume.size'].unit: GiB
$.metrics['network.incoming.bytes'].unit: MB
$.metrics['network.outgoing.bytes'].unit: MB
$.metrics['ip.floating'].unit: ip
$.metrics['radosgw.objects.size'].unit: GiB
- name: get metrics info
url: /v1/info/metrics
status: 200
response_json_paths:
$.services.`len`: 6
$.services[/service_id][0].service_id: compute
$.services[/service_id][0].unit: instance
$.services[/service_id][1].service_id: image
$.services[/service_id][1].unit: MiB
$.services[/service_id][2].service_id: network.bw.in
$.services[/service_id][2].unit: MB
$.services[/service_id][3].service_id: network.bw.out
$.services[/service_id][3].unit: MB
$.services[/service_id][4].service_id: network.floating
$.services[/service_id][4].unit: ip
$.services[/service_id][5].service_id: volume
$.services[/service_id][5].unit: GiB
$.metrics.`len`: 13
$.metrics[/metric_id][0].metric_id: cpu
$.metrics[/metric_id][0].unit: instance
$.metrics[/metric_id][1].metric_id: disk.ephemeral.size
$.metrics[/metric_id][1].unit: instance
$.metrics[/metric_id][2].metric_id: disk.root.size
$.metrics[/metric_id][2].unit: instance
$.metrics[/metric_id][3].metric_id: image.download
$.metrics[/metric_id][3].unit: MiB
$.metrics[/metric_id][4].metric_id: image.serve
$.metrics[/metric_id][4].unit: MiB
$.metrics[/metric_id][5].metric_id: image.size
$.metrics[/metric_id][5].unit: MiB
$.metrics[/metric_id][6].metric_id: ip.floating
$.metrics[/metric_id][6].unit: ip
$.metrics[/metric_id][7].metric_id: memory
$.metrics[/metric_id][7].unit: instance
$.metrics[/metric_id][8].metric_id: network.incoming.bytes
$.metrics[/metric_id][8].unit: MB
$.metrics[/metric_id][9].metric_id: network.outgoing.bytes
$.metrics[/metric_id][9].unit: MB
$.metrics[/metric_id][10].metric_id: radosgw.objects.size
$.metrics[/metric_id][10].unit: GiB
$.metrics[/metric_id][11].metric_id: vcpus
$.metrics[/metric_id][11].unit: instance
$.metrics[/metric_id][12].metric_id: volume.size
$.metrics[/metric_id][12].unit: GiB
- name: get compute service info
url: /v1/info/services/compute
- name: get cpu metric info
url: /v1/info/metrics/cpu
status: 200
response_json_paths:
$.service_id: compute
$.metric_id: cpu
$.unit: instance
$.metadata.`len`: 8
$.metadata.`len`: 4

78
cloudkitty/tests/samples.py
View File

@ -18,6 +18,7 @@
import copy
import decimal
from cloudkitty.default_metrics_conf import DEFAULT_METRICS_CONF
from cloudkitty import utils as ck_utils
TENANT = 'f266f30b11f246b589fd266f85eeec39'
@ -74,12 +75,12 @@ SECOND_PERIOD = {
COLLECTED_DATA = [{
'period': FIRST_PERIOD,
'usage': {
'compute': [{
'cpu': [{
'desc': COMPUTE_METADATA,
'vol': {
'qty': decimal.Decimal(1.0),
'unit': 'instance'}}],
'image': [{
'image.size': [{
'desc': IMAGE_METADATA,
'vol': {
'qty': decimal.Decimal(1.0),
@ -87,7 +88,7 @@ COLLECTED_DATA = [{
}}, {
'period': SECOND_PERIOD,
'usage': {
'compute': [{
'cpu': [{
'desc': COMPUTE_METADATA,
'vol': {
'qty': decimal.Decimal(1.0),
@ -95,11 +96,11 @@ COLLECTED_DATA = [{
}}]
RATED_DATA = copy.deepcopy(COLLECTED_DATA)
RATED_DATA[0]['usage']['compute'][0]['rating'] = {
RATED_DATA[0]['usage']['cpu'][0]['rating'] = {
'price': decimal.Decimal('0.42')}
RATED_DATA[0]['usage']['image'][0]['rating'] = {
RATED_DATA[0]['usage']['image.size'][0]['rating'] = {
'price': decimal.Decimal('0.1337')}
RATED_DATA[1]['usage']['compute'][0]['rating'] = {
RATED_DATA[1]['usage']['cpu'][0]['rating'] = {
'price': decimal.Decimal('0.42')}
@ -121,70 +122,13 @@ def split_storage_data(raw_data):
# FIXME(sheeprine): storage is not using decimal for rates, we need to
# transition to decimal.
STORED_DATA = copy.deepcopy(COLLECTED_DATA)
STORED_DATA[0]['usage']['compute'][0]['rating'] = {
STORED_DATA[0]['usage']['cpu'][0]['rating'] = {
'price': 0.42}
STORED_DATA[0]['usage']['image'][0]['rating'] = {
STORED_DATA[0]['usage']['image.size'][0]['rating'] = {
'price': 0.1337}
STORED_DATA[1]['usage']['compute'][0]['rating'] = {
STORED_DATA[1]['usage']['cpu'][0]['rating'] = {
'price': 0.42}
STORED_DATA = split_storage_data(STORED_DATA)
METRICS_CONF = {
'collector': 'gnocchi',
'name': 'OpenStack',
'period': 3600,
'services': [
'compute',
'volume',
'network.bw.in',
'network.bw.out',
'network.floating',
'image'
],
'services_metrics': {
'compute': [
{'vcpus': 'max'},
{'memory': 'max'},
{'cpu': 'max'},
{'disk.root.size': 'max'},
{'disk.ephemeral.size': 'max'}
],
'image': [
{'image.size': 'max'},
{'image.download': 'max'},
{'image.serve': 'max'}
],
'network.bw.in': [{'network.incoming.bytes': 'max'}],
'network.bw.out': [{'network.outgoing.bytes': 'max'}],
'network.floating': [{'ip.floating': 'max'}],
'volume': [{'volume.size': 'max'}],
'radosgw.usage': [{'radosgw.objects.size': 'max'}]},
'services_objects': {
'compute': 'instance',
'image': 'image',
'network.bw.in': 'instance_network_interface',
'network.bw.out': 'instance_network_interface',
'network.floating': 'network',
'volume': 'volume',
'radosgw.usage': 'ceph_account',
},
'metrics_units': {
'compute': {1: {'unit': 'instance'}},
'default_unit': {1: {'unit': 'unknown'}},
'image': {'image.size': {'unit': 'MiB', 'factor': '1/1048576'}},
'network.bw.in': {'network.incoming.bytes': {
'unit': 'MB',
'factor': '1/1000000'}},
'network.bw.out': {'network.outgoing.bytes': {
'unit': 'MB',
'factor': '1/1000000'}},
'network.floating': {1: {'unit': 'ip'}},
'volume': {'volume.size': {'unit': 'GiB'}},
'radosgw.usage': {'radosgw.objects.size': {
'unit': 'GiB',
'factor': '1/1073741824'}},
},
'wait_periods': 2,
'window': 1800
}
METRICS_CONF = DEFAULT_METRICS_CONF

18
cloudkitty/tests/storage/test_storage.py
View File

@ -234,10 +234,10 @@ class StorageTotalTest(StorageTest):
total = self.storage.get_total(
begin=begin,
end=end,
service='compute')
service='cpu')
self.assertEqual(1, len(total))
self.assertEqual(0.84, total[0]["rate"])
self.assertEqual('compute', total[0]["res_type"])
self.assertEqual('cpu', total[0]["res_type"])
self.assertEqual(begin, total[0]["begin"])
self.assertEqual(end, total[0]["end"])
@ -269,11 +269,11 @@ class StorageTotalTest(StorageTest):
groupby="res_type")
self.assertEqual(2, len(total))
self.assertEqual(0.2674, total[0]["rate"])
self.assertEqual('image', total[0]["res_type"])
self.assertEqual('image.size', total[0]["res_type"])
self.assertEqual(begin, total[0]["begin"])
self.assertEqual(end, total[0]["end"])
self.assertEqual(1.68, total[1]["rate"])
self.assertEqual('compute', total[1]["res_type"])
self.assertEqual('cpu', total[1]["res_type"])
self.assertEqual(begin, total[1]["begin"])
self.assertEqual(end, total[1]["end"])
@ -288,22 +288,22 @@ class StorageTotalTest(StorageTest):
self.assertEqual(4, len(total))
self.assertEqual(0.1337, total[0]["rate"])
self.assertEqual(self._other_tenant_id, total[0]["tenant_id"])
self.assertEqual('image', total[0]["res_type"])
self.assertEqual('image.size', total[0]["res_type"])
self.assertEqual(begin, total[0]["begin"])
self.assertEqual(end, total[0]["end"])
self.assertEqual(0.1337, total[1]["rate"])
self.assertEqual(self._tenant_id, total[1]["tenant_id"])
self.assertEqual('image', total[1]["res_type"])
self.assertEqual('image.size', total[1]["res_type"])
self.assertEqual(begin, total[1]["begin"])
self.assertEqual(end, total[1]["end"])
self.assertEqual(0.84, total[2]["rate"])
self.assertEqual(self._other_tenant_id, total[2]["tenant_id"])
self.assertEqual('compute', total[2]["res_type"])
self.assertEqual('cpu', total[2]["res_type"])
self.assertEqual(begin, total[2]["begin"])
self.assertEqual(end, total[2]["end"])
self.assertEqual(0.84, total[3]["rate"])
self.assertEqual(self._tenant_id, total[3]["tenant_id"])
self.assertEqual('compute', total[3]["res_type"])
self.assertEqual('cpu', total[3]["res_type"])
self.assertEqual(begin, total[3]["begin"])
self.assertEqual(end, total[3]["end"])
@ -426,7 +426,7 @@ class StorageDataIntegrityTest(StorageTest):
del expected_data[2]
# NOTE(sheeprine): Quick and dirty sort (ensure result consistency,
# order is not significant to the test result)
if 'image' in stored_data[0]['usage']:
if 'image.size' in stored_data[0]['usage']:
stored_data[0]['usage'], stored_data[1]['usage'] = (
stored_data[1]['usage'], stored_data[0]['usage'])
self.assertEqual(

6
cloudkitty/tests/test_keystone_fetcher.py
View File

@ -20,7 +20,7 @@ import unittest
import mock
from oslo_utils import uuidutils
from cloudkitty.tenant_fetcher import keystone
from cloudkitty.fetcher import keystone
from cloudkitty import tests
@ -72,10 +72,10 @@ class KeystoneFetcherTest(tests.TestCase):
super(KeystoneFetcherTest, self).setUp()
self.conf.set_override('backend', 'keystone', 'tenant_fetcher')
self.conf.import_group('keystone_fetcher',
'cloudkitty.tenant_fetcher.keystone')
'cloudkitty.fetcher.keystone')
@unittest.SkipTest
def test_keystone_tenant_fetcher_filter_list(self):
def test_keystone_fetcher_filter_list(self):
kclient = 'keystoneclient.client.Client'
with mock.patch(kclient) as kclientmock:
kclientmock.return_value = Client()

2
cloudkitty/tests/test_orchestrator.py
View File

@ -40,7 +40,7 @@ class OrchestratorTest(tests.TestCase):
messaging_conf.transport_driver = 'fake'
self.conf.set_override('backend', 'keystone', 'tenant_fetcher')
self.conf.import_group('keystone_fetcher',
'cloudkitty.tenant_fetcher.keystone')
'cloudkitty.fetcher.keystone')
def setup_fake_modules(self):
fake_module1 = tests.FakeRatingModule()

40
cloudkitty/utils.py
View File

@ -40,12 +40,18 @@ from six import moves
from stevedore import extension
COLLECTORS_NAMESPACE = 'cloudkitty.collector.backends'
_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f'
_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
LOG = logging.getLogger(__name__)
collect_opts = [
cfg.StrOpt('fetcher',
default='keystone',
deprecated_for_removal=True,
help='Project fetcher.'),
cfg.StrOpt('collector',
default='gnocchi',
deprecated_for_removal=True,
@ -77,8 +83,16 @@ collect_opts = [
default='/etc/cloudkitty/metrics.yml',
help='Metrology configuration file.'),
]
storage_opts = [
cfg.StrOpt('backend',
default='sqlalchemy',
help='Name of the storage backend driver.')
]
CONF = cfg.CONF
CONF.register_opts(collect_opts, 'collect')
CONF.register_opts(storage_opts, 'storage')
def isotime(at=None, subsecond=False):
@ -263,28 +277,20 @@ def check_time_state(timestamp=None, period=0, wait_time=0):
def get_metrics_conf(conf_path):
"""Return loaded yaml metrology configuration.
In case of empty /etc/cloudkitty folder,
a fallback is done on the former deprecated oslo config method.
In case not found metrics.yml file,
return an empty dict.
"""
res = None
# NOTE(mc): We can not raise any exception in this function as it called
# at some file imports. Default values should be used instead. This is
# done for the docs and tests in gerrit which does not copy yaml conf file.
try:
with open(conf_path) as conf:
res = yaml.safe_load(conf)
res = res[0]
except Exception as exc:
res.update({'storage': CONF.storage.backend})
return res or {}
except Exception:
LOG.warning('Error when trying to retrieve yaml metrology conf file.')
LOG.warning(exc)
LOG.warning('Fallback on the deprecated oslo config method.')
try:
res = {key: val for key, val in CONF.collect.items()}
except Exception as exc:
err_msg = 'Error when trying to retrieve ' \
'deprecated oslo config method.'
LOG.error(err_msg)
LOG.error(exc)
return res
return {}
@contextlib.contextmanager

17
doc/source/webapi/v1.rst
View File

@ -30,14 +30,17 @@ Info
.. rest-controller:: cloudkitty.api.v1.controllers.info:InfoController
:webprefix: /v1/info
.. rest-controller:: cloudkitty.api.v1.controllers.info:MetricInfoController
:webprefix: /v1/info/metric
.. autotype:: cloudkitty.api.v1.datamodels.info.CloudkittyMetricInfo
:members:
.. autotype:: cloudkitty.api.v1.datamodels.info.CloudkittyMetricInfoCollection
:members:
.. rest-controller:: cloudkitty.api.v1.controllers.info:ServiceInfoController
:webprefix: /v1/info/services
.. autotype:: cloudkitty.api.v1.datamodels.info.CloudkittyServiceInfo
:members:
.. autotype:: cloudkitty.api.v1.datamodels.info.CloudkittyServiceInfoCollection
:members:
:webprefix: /v1/info/service
Rating

167
etc/cloudkitty/metrics.yml
View File

@ -1,75 +1,102 @@
- name: OpenStack
name: OpenStack
collector: gnocchi
period: 3600
wait_periods: 2
window: 1800
fetcher: keystone
collector: gnocchi
services:
- compute
- volume
- network.bw.in
- network.bw.out
- network.floating
- image
period: 3600
wait_periods: 2
window: 1800
services_objects:
compute: instance
volume: volume
network.bw.out: instance_network_interface
network.bw.in: instance_network_interface
network.floating: network
image: image
radosgw.usage: ceph_account
services_objects:
compute: instance
volume: volume
network.bw.out: instance_network_interface
network.bw.in: instance_network_interface
network.floating: network
image: image
radosgw.usage: ceph_account
services_metrics:
compute:
- vcpus: max
- memory: max
- cpu: max
- disk.root.size: max
- disk.ephemeral.size: max
volume:
- volume.size: max
network.bw.in:
- network.incoming.bytes: max
network.bw.out:
- network.outgoing.bytes: max
network.floating:
- ip.floating: max
image:
- image.size: max
- image.download: max
- image.serve: max
radosgw.usage:
- radosgw.objects.size: max
metrics:
vcpus:
resource: instance
unit: instance
factor: 1
aggregation_method: max
countable_unit: true
metrics_units:
compute:
1:
unit: instance
volume:
volume.size:
unit: GiB
network.bw.in:
network.incoming.bytes:
unit: MB
factor: 1/1000000
network.bw.out:
network.outgoing.bytes:
unit: MB
factor: 1/1000000
network.floating:
1:
unit: ip
image:
image.size:
unit: MiB
factor: 1/1048576
radosgw.usage:
radosgw.objects.size:
unit: GiB
factor: 1/1073741824
default_unit:
1:
unit: unknown
memory:
resource: instance
unit: instance
factor: 1
aggregation_method: max
countable_unit: true
cpu:
resource: instance
unit: instance
factor: 1
aggregation_method: max
countable_unit: true
disk.root.size:
resource: instance
unit: instance
factor: 1
aggregation_method: max
countable_unit: true
disk.ephemeral.size:
resource: instance
unit: instance
factor: 1
aggregation_method: max
countable_unit: true
image.size:
resource: image
unit: MiB
factor: 1/1048576
aggregation_method: max
image.download:
resource: image
unit: MiB
factor: 1/1048576
aggregation_method: max
image.serve:
resource: image
unit: MiB
factor: 1/1048576
aggregation_method: max
volume.size:
resource: volume
unit: GiB
factor: 1
aggregation_method: max
network.outgoing.bytes:
resource: instance_network_interface
unit: MB
factor: 1/1000000
aggregation_method: max
network.incoming.bytes:
resource: instance_network_interface
unit: MB
factor: 1/1000000
aggregation_method: max
ip.floating:
resource: network
unit: ip
factor: 1
aggregation_method: max
countable_unit: true
radosgw.objects.size:
resource: ceph_account
unit: GiB
factor: 1/1073741824
aggregation_method: max

6
releasenotes/notes/change-metrology-organization-1e11900eb30780cc.yaml Normal file
View File

@ -0,0 +1,6 @@
---
other:
- |
Cloudkitty now bases itself on metrics and no more on services for data
valorization. The metrics.yml file information has been reorganized
accordingly.

8
releasenotes/notes/deprecate-info-services-endpoints-0c5018cb08a30d5f.yaml Normal file
View File

@ -0,0 +1,8 @@
---
deprecations:
- |
The /v1/info/services and /v1/info/services/<service> endpoints have
been deprecated. The /v1/info/metrics and /v1/info/metrics/<metric>
endpoints should be used instead.
The whole /v1/info API part is currently being reworked, and some
endpoints will also be deprecated and deleted in the future.

5
releasenotes/notes/source-fetcher-43c4352508f7f944.yaml Normal file
View File

@ -0,0 +1,5 @@
---
features:
- |
A Source Fetcher has been added, allowing to add new collectors to scrap
metrics from non-OpenStack sources.

5
setup.cfg
View File

@ -51,8 +51,9 @@ cloudkitty.collector.backends =
meta = cloudkitty.collector.meta:MetaCollector
cloudkitty.tenant.fetchers =
fake = cloudkitty.tenant_fetcher.fake:FakeFetcher
keystone = cloudkitty.tenant_fetcher.keystone:KeystoneFetcher
fake = cloudkitty.fetcher.fake:FakeFetcher
keystone = cloudkitty.fetcher.keystone:KeystoneFetcher
source = cloudkitty.fetcher.source:SourceFetcher
cloudkitty.transformers =
CloudKittyFormatTransformer = cloudkitty.transformer.format:CloudKittyFormatTransformer