openstack/cloudkitty
Code Issues Proposed changes

Split metrology configuration from CK config file

Browse Source 
To avoid source code updates with metrology conf update,
the metrology configuration is separated from Cloudkitty
configuration file and placed in a yaml one.

Task: 5724
Story: 2001215

Change-Id: Icc098c40bc52c2589e89d705d9d711d0ce2fb557
This commit is contained in:
Martin CAMEY 2017-09-26 11:49:05 +02:00 committed by MC
parent 7ff7910013
commit b8c848f9ec
18 changed files with 512 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cloudkitty/api/v1/controllers/info.py
View File

@ -26,8 +26,12 @@ from cloudkitty.api.v1.datamodels import info as info_models
from cloudkitty.api.v1 import types as ck_types
from cloudkitty import collector
from cloudkitty.common import policy
from cloudkitty import utils as ck_utils
CONF = cfg.CONF
METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
METADATA = collector.get_collector_metadata()
@ -79,5 +83,5 @@ class InfoController(rest.RestController):
"""Return current configuration."""
policy.enforce(pecan.request.context, 'info:get_config', {})
info = {}
info["collect"] = {key: value for key, value in CONF.collect.items()}
info["collect"] = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
return info

8
cloudkitty/api/v1/datamodels/info.py
View File

@ -18,10 +18,14 @@
from oslo_config import cfg
from wsme import types as wtypes
from cloudkitty import utils as ck_utils
CONF = cfg.CONF
CONF.import_opt('services', 'cloudkitty.collector', 'collect')
METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
CLOUDKITTY_SERVICES = wtypes.Enum(wtypes.text,
*CONF.collect.services)
*METRICS_CONF['services'])
class CloudkittyServiceInfo(wtypes.Base):

7
cloudkitty/api/v1/datamodels/rating.py
View File

@ -21,11 +21,14 @@ from oslo_config import cfg
from wsme import types as wtypes
from cloudkitty.api.v1 import types as cktypes
from cloudkitty import utils as ck_utils
CONF = cfg.CONF
CONF.import_opt('services', 'cloudkitty.collector', 'collect')
METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
CLOUDKITTY_SERVICES = wtypes.Enum(wtypes.text,
*CONF.collect.services)
*METRICS_CONF['services'])
class CloudkittyResource(wtypes.Base):

8
cloudkitty/cli/processor.py
View File

@ -15,12 +15,18 @@
#
# @author: Stéphane Albert
#
from cloudkitty import orchestrator
from cloudkitty import service
def main():
service.prepare_service()
# NOTE(mc): This import is done here to ensure that the prepare_service()
# fonction is called before any cfg option. By importing the orchestrator
# file, the utils one is imported too, and then some cfg option are read
# before the prepare_service(), making cfg.CONF returning default values
# systematically.
from cloudkitty import orchestrator
processor = orchestrator.Orchestrator()
try:
processor.process()

34
cloudkitty/collector/__init__.py
View File

@ -22,32 +22,12 @@ import six
from stevedore import driver
from cloudkitty import transformer
import cloudkitty.utils as ck_utils
from cloudkitty import utils as ck_utils
collect_opts = [
cfg.StrOpt('collector',
default='ceilometer',
help='Data collector.'),
cfg.IntOpt('window',
default=1800,
help='Number of samples to collect per call.'),
cfg.IntOpt('period',
default=3600,
help='Rating period in seconds.'),
cfg.IntOpt('wait_periods',
default=2,
help='Wait for N periods before collecting new data.'),
cfg.ListOpt('services',
default=['compute',
'image',
'volume',
'network.bw.in',
'network.bw.out',
'network.floating'],
help='Services to monitor.'), ]
CONF = cfg.CONF
CONF.register_opts(collect_opts, 'collect')
METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
COLLECTORS_NAMESPACE = 'cloudkitty.collector.backends'
@ -56,11 +36,11 @@ def get_collector(transformers=None):
if not transformers:
transformers = transformer.get_transformers()
collector_args = {
'period': CONF.collect.period,
'period': METRICS_CONF['period'],
'transformers': transformers}
collector = driver.DriverManager(
COLLECTORS_NAMESPACE,
CONF.collect.collector,
METRICS_CONF['collector'],
invoke_on_load=True,
invoke_kwds=collector_args).driver
return collector
@ -73,10 +53,10 @@ def get_collector_metadata():
"""
transformers = transformer.get_transformers()
collector = driver.DriverManager(
COLLECTORS_NAMESPACE, CONF.collect.collector,
COLLECTORS_NAMESPACE, METRICS_CONF['collector'],
invoke_on_load=False).driver
metadata = {}
for service in CONF.collect.services:
for service in METRICS_CONF['services']:
metadata[service] = collector.get_metadata(service, transformers)
return metadata

112
cloudkitty/collector/ceilometer.py
View File

@ -20,12 +20,15 @@ import decimal
from ceilometerclient import client as cclient
from keystoneauth1 import loading as ks_loading
from oslo_config import cfg
from oslo_log import log as logging
from oslo_utils import units
from cloudkitty import collector
from cloudkitty import utils as ck_utils
LOG = logging.getLogger(__name__)
CEILOMETER_COLLECTOR_OPTS = 'ceilometer_collector'
ceilometer_collector_opts = ks_loading.get_auth_common_conf_options()
@ -38,6 +41,8 @@ ks_loading.register_auth_conf_options(
CEILOMETER_COLLECTOR_OPTS)
CONF = cfg.CONF
METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
class ResourceNotFound(Exception):
"""Raised when the resource doesn't exist."""
@ -113,7 +118,15 @@ class CeilometerCollector(collector.BaseCollector):
try:
info["metadata"].extend(transformers['CeilometerTransformer']
.get_metadata(resource_name))
info["unit"] = cls.units_mappings[resource_name]
try:
info["unit"] = METRICS_CONF['services_units'][resource_name][1]
# NOTE(mc): deprecated except part kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
info["unit"] = cls.units_mappings[resource_name]
except KeyError:
pass
return info
@ -201,9 +214,22 @@ class CeilometerCollector(collector.BaseCollector):
instance)
instance = self._cacher.get_resource_detail('compute',
instance_id)
compute_data.append(
self.t_cloudkitty.format_item(instance, self.units_mappings[
"compute"], 1))
try:
compute_data.append(self.t_cloudkitty.format_item(
instance,
METRICS_CONF['services_units']['compute'],
1,
))
# NOTE(mc): deprecated except part kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
compute_data.append(self.t_cloudkitty.format_item(
instance,
self.units_mappings['compute'],
1,
))
if not compute_data:
raise collector.NoDataCollected(self.collector_name, 'compute')
return self.t_cloudkitty.format_service('compute', compute_data)
@ -228,9 +254,22 @@ class CeilometerCollector(collector.BaseCollector):
image_id)
image_size_mb = decimal.Decimal(image_stats.max) / units.Mi
image_data.append(
self.t_cloudkitty.format_item(image, self.units_mappings[
"image"], image_size_mb))
try:
image_data.append(self.t_cloudkitty.format_item(
image,
METRICS_CONF['services_units']['image'],
image_size_mb,
))
# NOTE(mc): deprecated except part kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
image_data.append(self.t_cloudkitty.format_item(
image,
self.units_mappings['image'],
image_size_mb,
))
if not image_data:
raise collector.NoDataCollected(self.collector_name, 'image')
@ -255,9 +294,23 @@ class CeilometerCollector(collector.BaseCollector):
volume)
volume = self._cacher.get_resource_detail('volume',
volume_id)
volume_data.append(
self.t_cloudkitty.format_item(volume, self.units_mappings[
"volume"], volume_stats.max))
try:
volume_data.append(self.t_cloudkitty.format_item(
volume,
METRICS_CONF['services_units']['volume'],
volume_stats.max,
))
# NOTE(mc): deprecated except part kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
volume_data.append(self.t_cloudkitty.format_item(
volume,
self.units_mappings['volume'],
volume_stats.max,
))
if not volume_data:
raise collector.NoDataCollected(self.collector_name, 'volume')
return self.t_cloudkitty.format_service('volume', volume_data)
@ -294,9 +347,22 @@ class CeilometerCollector(collector.BaseCollector):
tap_id)
tap_bw_mb = decimal.Decimal(tap_stat.max) / units.M
bw_data.append(
self.t_cloudkitty.format_item(tap, self.units_mappings[
"network.bw." + direction], tap_bw_mb))
try:
bw_data.append(self.t_cloudkitty.format_item(
tap,
METRICS_CONF['services_units']['network.bw.' + direction],
tap_bw_mb,
))
# NOTE(mc): deprecated except part kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
bw_data.append(self.t_cloudkitty.format_item(
tap,
self.units_mappings['network.bw.' + direction],
tap_bw_mb,
))
ck_res_name = 'network.bw.{}'.format(direction)
if not bw_data:
@ -342,9 +408,23 @@ class CeilometerCollector(collector.BaseCollector):
floating)
floating = self._cacher.get_resource_detail('network.floating',
floating_id)
floating_data.append(
self.t_cloudkitty.format_item(floating, self.units_mappings[
"network.floating"], 1))
try:
floating_data.append(self.t_cloudkitty.format_item(
floating,
METRICS_CONF['services_units']['network.floating'],
1,
))
# NOTE(mc): deprecated except part kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
floating_data.append(self.t_cloudkitty.format_item(
floating,
self.units_mappings['network.floating'],
1,
))
if not floating_data:
raise collector.NoDataCollected(self.collector_name,
'network.floating')

107
cloudkitty/collector/gnocchi.py
View File

@ -18,11 +18,15 @@ import decimal
from gnocchiclient import client as gclient
from keystoneauth1 import loading as ks_loading
from oslo_config import cfg
from oslo_log import log as logging
from oslo_utils import units
from cloudkitty import collector
from cloudkitty import utils as ck_utils
LOG = logging.getLogger(__name__)
GNOCCHI_COLLECTOR_OPTS = 'gnocchi_collector'
gnocchi_collector_opts = ks_loading.get_auth_common_conf_options()
@ -35,6 +39,8 @@ ks_loading.register_auth_conf_options(
GNOCCHI_COLLECTOR_OPTS)
CONF = cfg.CONF
METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
class GnocchiCollector(collector.BaseCollector):
collector_name = 'gnocchi'
@ -102,7 +108,15 @@ class GnocchiCollector(collector.BaseCollector):
try:
info["metadata"].extend(transformers['GnocchiTransformer']
.get_metadata(resource_name))
info["unit"] = cls.units_mappings[resource_name][1]
try:
info["unit"] = METRICS_CONF['services_units'][resource_name][1]
# NOTE(mc): deprecated except part kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
info["unit"] = cls.units_mappings[resource_name][1]
except KeyError:
pass
return info
@ -155,26 +169,50 @@ class GnocchiCollector(collector.BaseCollector):
self.gen_filter(cop="<=", started_at=end))
return time_filter
def _expand(self, metrics, resource, name, aggregate, start, end):
try:
values = self._conn.metric.get_measures(
metric=metrics[name],
start=ck_utils.ts2dt(start),
stop=ck_utils.ts2dt(end),
aggregation=aggregate)
# NOTE(sheeprine): Get the list of values for the current
# metric and get the first result value.
# [point_date, granularity, value]
# ["2015-11-24T00:00:00+00:00", 86400.0, 64.0]
resource[name] = values[0][2]
except IndexError:
resource[name] = 0
except KeyError:
# Skip metrics not found
pass
def _expand_metrics(self, resources, mappings, start, end):
for resource in resources:
metrics = resource.get('metrics', {})
for name, aggregate in mappings:
try:
values = self._conn.metric.get_measures(
metric=metrics[name],
start=ck_utils.ts2dt(start),
stop=ck_utils.ts2dt(end),
aggregation=aggregate)
# NOTE(sheeprine): Get the list of values for the current
# metric and get the first result value.
# [point_date, granularity, value]
# ["2015-11-24T00:00:00+00:00", 86400.0, 64.0]
resource[name] = values[0][2]
except IndexError:
resource[name] = 0
except KeyError:
# Skip metrics not found
pass
try:
for mapping in mappings:
self._expand(
metrics,
resource,
mapping.keys()[0],
mapping.values()[0],
start,
end,
)
# NOTE(mc): deprecated except part kept for backward compatibility.
except AttributeError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
for name, aggregate in mappings:
self._expand(
metrics,
resource,
name,
aggregate,
start,
end,
)
def get_resources(self, resource_name, start, end,
project_id, q_filter=None):
@ -196,7 +234,15 @@ class GnocchiCollector(collector.BaseCollector):
# Translating the resource name if needed
query_parameters = self._generate_time_filter(start, end)
resource_type = self.retrieve_mappings.get(resource_name)
try:
resource_type = METRICS_CONF['services_objects'].get(resource_name)
# NOTE(mc): deprecated except part kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
resource_type = self.retrieve_mappings.get(resource_name)
query_parameters.append(
self.gen_filter(cop="=", type=resource_type))
query_parameters.append(
@ -210,7 +256,18 @@ class GnocchiCollector(collector.BaseCollector):
def resource_info(self, resource_name, start, end, project_id,
q_filter=None):
qty, unit = self.units_mappings.get(resource_name, self.default_unit)
try:
qty = METRICS_CONF['services_units'][resource_name].keys()[0]
unit = METRICS_CONF['services_units'][resource_name].values()[0]
# NOTE(mc): deprecated except part kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
qty, unit = self.units_mappings.get(
resource_name,
self.default_unit,
)
resources = self.get_resources(resource_name, start, end,
project_id=project_id,
q_filter=q_filter)
@ -218,7 +275,15 @@ class GnocchiCollector(collector.BaseCollector):
for resource in resources:
resource_data = self.t_gnocchi.strip_resource_data(
resource_name, resource)
mappings = self.metrics_mappings[resource_name]
try:
mappings = METRICS_CONF['services_metrics'][resource_name]
# NOTE(mc): deprecated except part kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
mappings = self.metrics_mappings[resource_name]
self._expand_metrics([resource_data], mappings, start, end)
resource_data.pop('metrics', None)
# Convert network.bw.in, network.bw.out and image unit to MB

91
cloudkitty/collector/monasca.py
View File

@ -21,12 +21,16 @@ from keystoneauth1 import loading as ks_loading
from keystoneclient.v3 import client as ks_client
from monascaclient import client as mclient
from oslo_config import cfg
from oslo_log import log as logging
from oslo_utils import units
from cloudkitty import collector
from cloudkitty import transformer
from cloudkitty import utils as ck_utils
LOG = logging.getLogger(__name__)
MONASCA_API_VERSION = '2_0'
COLLECTOR_MONASCA_OPTS = 'collector_monasca'
collector_monasca_opts = ks_loading.get_auth_common_conf_options()
@ -40,6 +44,8 @@ ks_loading.register_auth_conf_options(
COLLECTOR_MONASCA_OPTS)
CONF = cfg.CONF
METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
class EndpointNotFound(Exception):
"""Exception raised if the Monasca endpoint is not found"""
@ -57,7 +63,7 @@ class MonascaCollector(collector.BaseCollector):
'network.bw.in': 'network.incoming.bytes',
'network.bw.out': 'network.outgoing.bytes',
}
metric_mappings = {
metrics_mappings = {
'compute': [
('cpu', 'max'),
('vpcus', 'max'),
@ -77,7 +83,7 @@ class MonascaCollector(collector.BaseCollector):
}
# (qty, unit). qty must be either a metric name, an integer
# or a decimal.Decimal object
unit_mappings = {
units_mappings = {
'compute': (1, 'instance'),
'image': ('image.size', 'MB'),
'volume': ('volume.size', 'GB'),
@ -127,9 +133,16 @@ class MonascaCollector(collector.BaseCollector):
def _get_metadata(self, resource_type, transformers):
info = {}
try:
info['unit'] = self.unit_mappings[resource_type][1]
except (KeyError, IndexError):
info['unit'] = self.default_unit[1]
info['unit'] = METRICS_CONF['services_units']
# NOTE(mc): deprecated second try kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
try:
info['unit'] = self.units_mappings[resource_type][1]
except (KeyError, IndexError):
info['unit'] = self.default_unit[1]
start = ck_utils.dt2ts(ck_utils.get_month_start())
end = ck_utils.dt2ts(ck_utils.get_month_end())
try:
@ -140,11 +153,19 @@ class MonascaCollector(collector.BaseCollector):
metadata = self._get_resource_metadata(resource_type, start,
end, resource_id)
info['metadata'] = metadata.keys()
try:
for metric, statistics in self.metric_mappings[resource_type]:
for metric, statistics in METRICS_CONF['services_metrics']:
info['metadata'].append(metric)
except (KeyError, IndexError):
pass
# NOTE(mc): deprecated second try kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
try:
for metric, statistics in self.metrics_mappings[resource_type]:
info['metadata'].append(metric)
except (KeyError, IndexError):
pass
return info
# NOTE(lukapeschke) if anyone sees a better way to do this,
@ -158,7 +179,14 @@ class MonascaCollector(collector.BaseCollector):
return tmp._get_metadata(resource_type, transformers)
def _get_resource_metadata(self, resource_type, start, end, resource_id):
meter = self.retrieve_mappings.get(resource_type)
try:
meter = METRICS_CONF['services_objects'].get(resource_type)
# NOTE(mc): deprecated except part kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
meter = self.retrieve_mappings.get(resource_type)
if not meter:
return {}
measurements = self._conn.metrics.list_measurements(
@ -209,7 +237,14 @@ class MonascaCollector(collector.BaseCollector):
def active_resources(self, resource_type, start,
end, project_id, **kwargs):
meter = self.retrieve_mappings.get(resource_type)
try:
meter = METRICS_CONF['services_objects'].get(resource_type)
# NOTE(mc): deprecated except part kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
meter = self.retrieve_mappings.get(resource_type)
if not meter:
return {}
dimensions = {}
@ -245,7 +280,21 @@ class MonascaCollector(collector.BaseCollector):
def resource_info(self, resource_type, start, end,
project_id, q_filter=None):
qty, unit = self.unit_mappings.get(resource_type, self.default_unit)
try:
qty, unit = METRICS_CONF['services_units'].get(
resource_type,
self.default_unit
)
# NOTE(mc): deprecated except part kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
qty, unit = self.units_mappings.get(
resource_type,
self.default_unit
)
active_resource_ids = self.active_resources(
resource_type, start, end, project_id
)
@ -253,11 +302,27 @@ class MonascaCollector(collector.BaseCollector):
for resource_id in active_resource_ids:
data = self._get_resource_metadata(resource_type, start,
end, resource_id)
mappings = self.metric_mappings[resource_type]
try:
mappings = METRICS_CONF['services_metrics'][resource_type]
# NOTE(mc): deprecated except part kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
mappings = self.metrics_mappings[resource_type]
self._expand_metrics(data, resource_id, mappings, start, end)
resource_qty = qty
if not (isinstance(qty, int) or isinstance(qty, decimal.Decimal)):
resource_qty = data[self.retrieve_mappings[resource_type]]
try:
resource_qty = METRICS_CONF['services_objects']
# NOTE(mc): deprecated except part kept for backward compat.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf')
msg = 'Fallback on the deprecated oslo config method'
LOG.warning(msg)
resource_qty = data[self.retrieve_mappings[resource_type]]
resource = self.t_cloudkitty.format_item(data, unit, resource_qty)
resource['desc']['resource_id'] = resource_id
resource['resource_id'] = resource_id

4
cloudkitty/common/config.py
View File

@ -17,7 +17,6 @@ import copy
import itertools
import cloudkitty.api.app
import cloudkitty.collector
import cloudkitty.collector.ceilometer
import cloudkitty.collector.gnocchi
import cloudkitty.collector.monasca
@ -28,6 +27,7 @@ import cloudkitty.storage
import cloudkitty.storage.gnocchi
import cloudkitty.tenant_fetcher
import cloudkitty.tenant_fetcher.keystone
import cloudkitty.utils
__all__ = ['list_opts']
@ -35,7 +35,7 @@ _opts = [
('api', list(itertools.chain(
cloudkitty.api.app.api_opts,))),
('collect', list(itertools.chain(
cloudkitty.collector.collect_opts))),
cloudkitty.utils.collect_opts))),
('ceilometer_collector', list(itertools.chain(
cloudkitty.collector.ceilometer.ceilometer_collector_opts))),
('collector_monasca', list(itertools.chain(

12
cloudkitty/orchestrator.py
View File

@ -51,6 +51,8 @@ orchestrator_opts = [
]
CONF.register_opts(orchestrator_opts, group='orchestrator')
METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
FETCHERS_NAMESPACE = 'cloudkitty.tenant.fetchers'
PROCESSORS_NAMESPACE = 'cloudkitty.rating.processors'
@ -152,8 +154,8 @@ class Worker(BaseWorker):
def __init__(self, collector, storage, tenant_id=None):
self._collector = collector
self._storage = storage
self._period = CONF.collect.period
self._wait_time = CONF.collect.wait_periods * self._period
self._period = METRICS_CONF['period']
self._wait_time = METRICS_CONF['wait_periods'] * self._period
super(Worker, self).__init__(tenant_id)
@ -180,7 +182,7 @@ class Worker(BaseWorker):
if not timestamp:
break
for service in CONF.collect.services:
for service in METRICS_CONF['services']:
try:
try:
data = self._collect(service, timestamp)
@ -232,8 +234,8 @@ class Orchestrator(object):
uuidutils.generate_uuid().encode('ascii'))
self.coord.start()
self._period = CONF.collect.period
self._wait_time = CONF.collect.wait_periods * self._period
self._period = METRICS_CONF['period']
self._wait_time = METRICS_CONF['wait_periods'] * self._period
def _lock(self, tenant_id):
lock_name = b"cloudkitty-" + str(tenant_id).encode('ascii')

6
cloudkitty/storage/__init__.py
View File

@ -33,12 +33,14 @@ storage_opts = [
CONF = cfg.CONF
CONF.register_opts(storage_opts, group='storage')
METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
STORAGES_NAMESPACE = 'cloudkitty.storage.backends'
def get_storage(collector=None):
storage_args = {
'period': CONF.collect.period,
'period': METRICS_CONF['period'],
'collector': collector if collector else ck_collector.get_collector()}
backend = driver.DriverManager(
STORAGES_NAMESPACE,
@ -63,7 +65,7 @@ class BaseStorage(object):
Handle incoming data from the global orchestrator, and store them.
"""
def __init__(self, **kwargs):
self._period = kwargs.get('period', CONF.collect.period)
self._period = kwargs.get('period', METRICS_CONF['period'])
self._collector = kwargs.get('collector')
# State vars

6
cloudkitty/storage/gnocchi/__init__.py
View File

@ -34,7 +34,7 @@ from cloudkitty import utils as ck_utils
LOG = log.getLogger(__name__)
CONF = cfg.CONF
CONF.import_opt('period', 'cloudkitty.collector', 'collect')
METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
GNOCCHI_STORAGE_OPTS = 'storage_gnocchi'
gnocchi_storage_opts = [
@ -44,7 +44,7 @@ gnocchi_storage_opts = [
# The archive policy definition MUST include the collect period granularity
cfg.StrOpt('archive_policy_definition',
default='[{"granularity": '
+ six.text_type(CONF.collect.period) +
+ six.text_type(METRICS_CONF['period']) +
', "timespan": "90 days"}, '
'{"granularity": 86400, "timespan": "360 days"}, '
'{"granularity": 2592000, "timespan": "1800 days"}]',
@ -84,7 +84,7 @@ class GnocchiStorage(storage.BaseStorage):
CONF.storage_gnocchi.archive_policy_name)
self._archive_policy_definition = json.loads(
CONF.storage_gnocchi.archive_policy_definition)
self._period = CONF.collect.period
self._period = METRICS_CONF['period']
if "period" in kwargs:
self._period = kwargs["period"]

14
cloudkitty/tests/gabbi/gabbits/v1-info.yaml
View File

@ -8,12 +8,12 @@ tests:
response_json_paths:
$.collect.services.`len`: 6
$.collect.services[0]: compute
$.collect.services[1]: image
$.collect.services[2]: volume
$.collect.services[3]: network.bw.in
$.collect.services[4]: network.bw.out
$.collect.services[5]: network.floating
$.collect.collector: ceilometer
$.collect.services[1]: volume
$.collect.services[2]: network.bw.in
$.collect.services[3]: network.bw.out
$.collect.services[4]: network.floating
$.collect.services[5]: image
$.collect.collector: gnocchi
$.collect.window: 1800
$.collect.wait_periods: 2
$.collect.period: 3600
@ -42,4 +42,4 @@ tests:
response_json_paths:
$.service_id: compute
$.unit: instance
$.metadata.`len`: 10
$.metadata.`len`: 8

67
cloudkitty/utils.py
View File

@ -24,7 +24,10 @@ to ease maintenance in case of library modifications.
import calendar
import datetime
import sys
import yaml
from oslo_config import cfg
from oslo_log import log as logging
from oslo_utils import timeutils
from six import moves
from stevedore import extension
@ -33,6 +36,43 @@ from stevedore import extension
_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f'
_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
LOG = logging.getLogger(__name__)
collect_opts = [
cfg.StrOpt('collector',
default='gnocchi',
deprecated_for_removal=True,
help='Data collector.'),
cfg.IntOpt('window',
default=1800,
deprecated_for_removal=True,
help='Number of samples to collect per call.'),
cfg.IntOpt('period',
default=3600,
deprecated_for_removal=True,
help='Rating period in seconds.'),
cfg.IntOpt('wait_periods',
default=2,
deprecated_for_removal=True,
help='Wait for N periods before collecting new data.'),
cfg.ListOpt('services',
default=[
'compute',
'volume',
'network.bw.in',
'network.bw.out',
'network.floating',
'image',
],
deprecated_for_removal=True,
help='Services to monitor.'),
cfg.StrOpt('metrics_conf',
default='/etc/cloudkitty/metrics.yml',
help='Metrology configuration file.'),
]
CONF = cfg.CONF
CONF.register_opts(collect_opts, 'collect')
def isotime(at=None, subsecond=False):
"""Stringify time in ISO 8601 format."""
@ -190,3 +230,30 @@ def check_time_state(timestamp=None, period=0, wait_time=0):
if next_timestamp + wait_time < now:
return next_timestamp
return 0
def get_metrics_conf(conf_path):
"""Return loaded yaml metrology configuration.
In case of empty /etc/cloudkitty folder,
a fallback is done on the former deprecated oslo config method.
"""
res = None
try:
with open(conf_path) as conf:
res = yaml.load(conf)
res = res[0]
except Exception as exc:
LOG.warning('Error when trying to retrieve yaml metrology conf file.')
LOG.warning(exc)
LOG.warning('Fallback on the deprecated oslo config method.')
try:
res = {key: val for key, val in CONF.collect.items()}
except Exception as exc:
err_msg = 'Error when trying to retrieve ' \
'deprecated oslo config method.'
LOG.error(err_msg)
LOG.error(exc)
return res

2
devstack/plugin.sh
View File

@ -134,6 +134,7 @@ function configure_cloudkitty {
cp $CLOUDKITTY_DIR$CLOUDKITTY_CONF_DIR/policy.json $CLOUDKITTY_CONF_DIR
cp $CLOUDKITTY_DIR$CLOUDKITTY_CONF_DIR/api_paste.ini $CLOUDKITTY_CONF_DIR
cp $CLOUDKITTY_DIR$CLOUDKITTY_CONF_DIR/metrics.yml $CLOUDKITTY_CONF_DIR
iniset_rpc_backend cloudkitty $CLOUDKITTY_CONF DEFAULT
iniset $CLOUDKITTY_CONF DEFAULT notification_topics 'notifications'
@ -161,6 +162,7 @@ function configure_cloudkitty {
iniset $CLOUDKITTY_CONF collect collector $CLOUDKITTY_COLLECTOR
iniset $CLOUDKITTY_CONF ${CLOUDKITTY_COLLECTOR}_collector auth_section authinfos
iniset $CLOUDKITTY_CONF collect services $CLOUDKITTY_SERVICES
iniset $CLOUDKITTY_CONF collect metrics_conf $CLOUDKITTY_CONF_DIR/$CLOUDKITTY_METRICS_CONF
# output
iniset $CLOUDKITTY_CONF output backend $CLOUDKITTY_OUTPUT_BACKEND

1
devstack/settings
View File

@ -43,6 +43,7 @@ CLOUDKITTY_PRICING_TENANT=${CLOUDKITTY_PRICING_TENANT:-"demo"}
# Set CloudKitty collect info
CLOUDKITTY_COLLECTOR=${CLOUDKITTY_COLLECTOR:-ceilometer}
CLOUDKITTY_SERVICES=${CLOUDKITTY_SERVICES:-compute}
CLOUDKITTY_METRICS_CONF=metrics.yml
# Set CloudKitty output info
CLOUDKITTY_OUTPUT_BACKEND=${CLOUDKITTY_OUTPUT_BACKEND:-"cloudkitty.backend.file.FileBackend"}

77
doc/source/configuration/index.rst
View File

@ -167,18 +167,83 @@ Three collectors are available: Ceilometer (deprecated, see the Telemetry
documentation), Gnocchi and Monasca. The Monasca collector collects metrics
published by the Ceilometer agent to Monasca using Ceilosca_.
The collect information, is separated from the Cloudkitty configuration file, in a yaml one.
This allows Cloudkitty users to change metrology configuration,
without modifying source code or Cloudkitty configuration file.
.. code-block:: ini
[collect]
collector = gnocchi
# Metrics are collected every 3600 seconds
period = 3600
# By default, only the compute service is enabled
services = compute, volume, network.bw.in, network.bw.out, network.floating, image
metrics_conf = /etc/cloudkitty/metrics.yml
[gnocchi_collector]
auth_section = ks_auth
The ``/etc/cloudkitty/metrics.yml`` file looks like this:
.. code-block:: yaml
- name: OpenStack
collector: gnocchi
period: 3600
wait_period: 2
window: 1800
services:
- compute
- volume
- network.bw.in
- network.bw.out
- network.floating
- image
services_objects:
compute: instance
volume: volume
network.bw.in: instance_network_interface
network.bw.out: instance_network_interface
network.floating: network
image: image
services_metrics:
compute:
- vcpus: max
- memory: max
- cpu: max
- disk.root.size: max
- disk.ephemeral.size: max
volume:
- volume.size: max
network.bw.in:
- network.incoming.bytes: max
network.bw.out:
- network.outgoing.bytes: max
network.floating:
- ip.floating: max
image:
- image.size: max
- image.download: max
- image.serve: max
services_units:
compute:
1: instance
volume:
volume.size: GB
network.bw.in:
network.incoming.bytes: MB
network.bw.out:
network.outgoing.bytes: MB
network.floating:
1: ip
image:
image.size: MB
default_unit:
1: unknown
Setup the database and storage backend
--------------------------------------
@ -274,4 +339,4 @@ Choose and start the API server
$ cloudkitty-api -p 8889
.. _Ceilosca: https://github.com/openstack/monasca-ceilometer
.. _Ceilosca: https://github.com/openstack/monasca-ceilometer

58
etc/cloudkitty/metrics.yml Normal file
View File

@ -0,0 +1,58 @@
- name: OpenStack
collector: gnocchi
period: 3600
wait_periods: 2
window: 1800
services:
- compute
- volume
- network.bw.in
- network.bw.out
- network.floating
- image
services_objects:
compute: instance
volume: volume
network.bw.out: instance_network_interface
network.bw.in: instance_network_interface
network.floating: network
image: image
services_metrics:
compute:
- vcpus: max
- memory: max
- cpu: max
- disk.root.size: max
- disk.ephemeral.size: max
volume:
- volume.size: max
network.bw.in:
- network.incoming: max
network.bw.out:
- network.outgoing.bytes: max
network.floating:
- ip.floating: max
image:
- image.size: max
- image.download: max
- image.serve: max
services_units:
compute:
1: instance
volume:
volume.size: GB
network.bw.in:
network.incoming.bytes: MB
network.bw.out:
network.outgoing.bytes: MB
network.floating:
1: ip
image:
image.size: MB
default_unit:
1: unknown