Split metrology configuration from CK config file
To avoid source code updates with metrology conf update, the metrology configuration is separated from Cloudkitty configuration file and placed in a yaml one. Task: 5724 Story: 2001215 Change-Id: Icc098c40bc52c2589e89d705d9d711d0ce2fb557
This commit is contained in:
parent
7ff7910013
commit
b8c848f9ec
@ -26,8 +26,12 @@ from cloudkitty.api.v1.datamodels import info as info_models
|
|||||||
from cloudkitty.api.v1 import types as ck_types
|
from cloudkitty.api.v1 import types as ck_types
|
||||||
from cloudkitty import collector
|
from cloudkitty import collector
|
||||||
from cloudkitty.common import policy
|
from cloudkitty.common import policy
|
||||||
|
from cloudkitty import utils as ck_utils
|
||||||
|
|
||||||
CONF = cfg.CONF
|
CONF = cfg.CONF
|
||||||
|
|
||||||
|
METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
|
||||||
|
|
||||||
METADATA = collector.get_collector_metadata()
|
METADATA = collector.get_collector_metadata()
|
||||||
|
|
||||||
|
|
||||||
@ -79,5 +83,5 @@ class InfoController(rest.RestController):
|
|||||||
"""Return current configuration."""
|
"""Return current configuration."""
|
||||||
policy.enforce(pecan.request.context, 'info:get_config', {})
|
policy.enforce(pecan.request.context, 'info:get_config', {})
|
||||||
info = {}
|
info = {}
|
||||||
info["collect"] = {key: value for key, value in CONF.collect.items()}
|
info["collect"] = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
|
||||||
return info
|
return info
|
||||||
|
@ -18,10 +18,14 @@
|
|||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
from wsme import types as wtypes
|
from wsme import types as wtypes
|
||||||
|
|
||||||
|
from cloudkitty import utils as ck_utils
|
||||||
|
|
||||||
CONF = cfg.CONF
|
CONF = cfg.CONF
|
||||||
CONF.import_opt('services', 'cloudkitty.collector', 'collect')
|
|
||||||
|
METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
|
||||||
|
|
||||||
CLOUDKITTY_SERVICES = wtypes.Enum(wtypes.text,
|
CLOUDKITTY_SERVICES = wtypes.Enum(wtypes.text,
|
||||||
*CONF.collect.services)
|
*METRICS_CONF['services'])
|
||||||
|
|
||||||
|
|
||||||
class CloudkittyServiceInfo(wtypes.Base):
|
class CloudkittyServiceInfo(wtypes.Base):
|
||||||
|
@ -21,11 +21,14 @@ from oslo_config import cfg
|
|||||||
from wsme import types as wtypes
|
from wsme import types as wtypes
|
||||||
|
|
||||||
from cloudkitty.api.v1 import types as cktypes
|
from cloudkitty.api.v1 import types as cktypes
|
||||||
|
from cloudkitty import utils as ck_utils
|
||||||
|
|
||||||
CONF = cfg.CONF
|
CONF = cfg.CONF
|
||||||
CONF.import_opt('services', 'cloudkitty.collector', 'collect')
|
|
||||||
|
METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
|
||||||
|
|
||||||
CLOUDKITTY_SERVICES = wtypes.Enum(wtypes.text,
|
CLOUDKITTY_SERVICES = wtypes.Enum(wtypes.text,
|
||||||
*CONF.collect.services)
|
*METRICS_CONF['services'])
|
||||||
|
|
||||||
|
|
||||||
class CloudkittyResource(wtypes.Base):
|
class CloudkittyResource(wtypes.Base):
|
||||||
|
@ -15,12 +15,18 @@
|
|||||||
#
|
#
|
||||||
# @author: Stéphane Albert
|
# @author: Stéphane Albert
|
||||||
#
|
#
|
||||||
from cloudkitty import orchestrator
|
|
||||||
from cloudkitty import service
|
from cloudkitty import service
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
service.prepare_service()
|
service.prepare_service()
|
||||||
|
|
||||||
|
# NOTE(mc): This import is done here to ensure that the prepare_service()
|
||||||
|
# fonction is called before any cfg option. By importing the orchestrator
|
||||||
|
# file, the utils one is imported too, and then some cfg option are read
|
||||||
|
# before the prepare_service(), making cfg.CONF returning default values
|
||||||
|
# systematically.
|
||||||
|
from cloudkitty import orchestrator
|
||||||
processor = orchestrator.Orchestrator()
|
processor = orchestrator.Orchestrator()
|
||||||
try:
|
try:
|
||||||
processor.process()
|
processor.process()
|
||||||
|
@ -22,32 +22,12 @@ import six
|
|||||||
from stevedore import driver
|
from stevedore import driver
|
||||||
|
|
||||||
from cloudkitty import transformer
|
from cloudkitty import transformer
|
||||||
import cloudkitty.utils as ck_utils
|
from cloudkitty import utils as ck_utils
|
||||||
|
|
||||||
collect_opts = [
|
|
||||||
cfg.StrOpt('collector',
|
|
||||||
default='ceilometer',
|
|
||||||
help='Data collector.'),
|
|
||||||
cfg.IntOpt('window',
|
|
||||||
default=1800,
|
|
||||||
help='Number of samples to collect per call.'),
|
|
||||||
cfg.IntOpt('period',
|
|
||||||
default=3600,
|
|
||||||
help='Rating period in seconds.'),
|
|
||||||
cfg.IntOpt('wait_periods',
|
|
||||||
default=2,
|
|
||||||
help='Wait for N periods before collecting new data.'),
|
|
||||||
cfg.ListOpt('services',
|
|
||||||
default=['compute',
|
|
||||||
'image',
|
|
||||||
'volume',
|
|
||||||
'network.bw.in',
|
|
||||||
'network.bw.out',
|
|
||||||
'network.floating'],
|
|
||||||
help='Services to monitor.'), ]
|
|
||||||
|
|
||||||
CONF = cfg.CONF
|
CONF = cfg.CONF
|
||||||
CONF.register_opts(collect_opts, 'collect')
|
|
||||||
|
METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
|
||||||
|
|
||||||
COLLECTORS_NAMESPACE = 'cloudkitty.collector.backends'
|
COLLECTORS_NAMESPACE = 'cloudkitty.collector.backends'
|
||||||
|
|
||||||
@ -56,11 +36,11 @@ def get_collector(transformers=None):
|
|||||||
if not transformers:
|
if not transformers:
|
||||||
transformers = transformer.get_transformers()
|
transformers = transformer.get_transformers()
|
||||||
collector_args = {
|
collector_args = {
|
||||||
'period': CONF.collect.period,
|
'period': METRICS_CONF['period'],
|
||||||
'transformers': transformers}
|
'transformers': transformers}
|
||||||
collector = driver.DriverManager(
|
collector = driver.DriverManager(
|
||||||
COLLECTORS_NAMESPACE,
|
COLLECTORS_NAMESPACE,
|
||||||
CONF.collect.collector,
|
METRICS_CONF['collector'],
|
||||||
invoke_on_load=True,
|
invoke_on_load=True,
|
||||||
invoke_kwds=collector_args).driver
|
invoke_kwds=collector_args).driver
|
||||||
return collector
|
return collector
|
||||||
@ -73,10 +53,10 @@ def get_collector_metadata():
|
|||||||
"""
|
"""
|
||||||
transformers = transformer.get_transformers()
|
transformers = transformer.get_transformers()
|
||||||
collector = driver.DriverManager(
|
collector = driver.DriverManager(
|
||||||
COLLECTORS_NAMESPACE, CONF.collect.collector,
|
COLLECTORS_NAMESPACE, METRICS_CONF['collector'],
|
||||||
invoke_on_load=False).driver
|
invoke_on_load=False).driver
|
||||||
metadata = {}
|
metadata = {}
|
||||||
for service in CONF.collect.services:
|
for service in METRICS_CONF['services']:
|
||||||
metadata[service] = collector.get_metadata(service, transformers)
|
metadata[service] = collector.get_metadata(service, transformers)
|
||||||
return metadata
|
return metadata
|
||||||
|
|
||||||
|
@ -20,12 +20,15 @@ import decimal
|
|||||||
from ceilometerclient import client as cclient
|
from ceilometerclient import client as cclient
|
||||||
from keystoneauth1 import loading as ks_loading
|
from keystoneauth1 import loading as ks_loading
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
|
from oslo_log import log as logging
|
||||||
from oslo_utils import units
|
from oslo_utils import units
|
||||||
|
|
||||||
from cloudkitty import collector
|
from cloudkitty import collector
|
||||||
from cloudkitty import utils as ck_utils
|
from cloudkitty import utils as ck_utils
|
||||||
|
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
CEILOMETER_COLLECTOR_OPTS = 'ceilometer_collector'
|
CEILOMETER_COLLECTOR_OPTS = 'ceilometer_collector'
|
||||||
ceilometer_collector_opts = ks_loading.get_auth_common_conf_options()
|
ceilometer_collector_opts = ks_loading.get_auth_common_conf_options()
|
||||||
|
|
||||||
@ -38,6 +41,8 @@ ks_loading.register_auth_conf_options(
|
|||||||
CEILOMETER_COLLECTOR_OPTS)
|
CEILOMETER_COLLECTOR_OPTS)
|
||||||
CONF = cfg.CONF
|
CONF = cfg.CONF
|
||||||
|
|
||||||
|
METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
|
||||||
|
|
||||||
|
|
||||||
class ResourceNotFound(Exception):
|
class ResourceNotFound(Exception):
|
||||||
"""Raised when the resource doesn't exist."""
|
"""Raised when the resource doesn't exist."""
|
||||||
@ -113,7 +118,15 @@ class CeilometerCollector(collector.BaseCollector):
|
|||||||
try:
|
try:
|
||||||
info["metadata"].extend(transformers['CeilometerTransformer']
|
info["metadata"].extend(transformers['CeilometerTransformer']
|
||||||
.get_metadata(resource_name))
|
.get_metadata(resource_name))
|
||||||
info["unit"] = cls.units_mappings[resource_name]
|
|
||||||
|
try:
|
||||||
|
info["unit"] = METRICS_CONF['services_units'][resource_name][1]
|
||||||
|
# NOTE(mc): deprecated except part kept for backward compatibility.
|
||||||
|
except KeyError:
|
||||||
|
LOG.warning('Error when trying to use yaml metrology conf.')
|
||||||
|
LOG.warning('Fallback on the deprecated oslo config method.')
|
||||||
|
info["unit"] = cls.units_mappings[resource_name]
|
||||||
|
|
||||||
except KeyError:
|
except KeyError:
|
||||||
pass
|
pass
|
||||||
return info
|
return info
|
||||||
@ -201,9 +214,22 @@ class CeilometerCollector(collector.BaseCollector):
|
|||||||
instance)
|
instance)
|
||||||
instance = self._cacher.get_resource_detail('compute',
|
instance = self._cacher.get_resource_detail('compute',
|
||||||
instance_id)
|
instance_id)
|
||||||
compute_data.append(
|
|
||||||
self.t_cloudkitty.format_item(instance, self.units_mappings[
|
try:
|
||||||
"compute"], 1))
|
compute_data.append(self.t_cloudkitty.format_item(
|
||||||
|
instance,
|
||||||
|
METRICS_CONF['services_units']['compute'],
|
||||||
|
1,
|
||||||
|
))
|
||||||
|
# NOTE(mc): deprecated except part kept for backward compatibility.
|
||||||
|
except KeyError:
|
||||||
|
LOG.warning('Error when trying to use yaml metrology conf.')
|
||||||
|
LOG.warning('Fallback on the deprecated oslo config method.')
|
||||||
|
compute_data.append(self.t_cloudkitty.format_item(
|
||||||
|
instance,
|
||||||
|
self.units_mappings['compute'],
|
||||||
|
1,
|
||||||
|
))
|
||||||
if not compute_data:
|
if not compute_data:
|
||||||
raise collector.NoDataCollected(self.collector_name, 'compute')
|
raise collector.NoDataCollected(self.collector_name, 'compute')
|
||||||
return self.t_cloudkitty.format_service('compute', compute_data)
|
return self.t_cloudkitty.format_service('compute', compute_data)
|
||||||
@ -228,9 +254,22 @@ class CeilometerCollector(collector.BaseCollector):
|
|||||||
image_id)
|
image_id)
|
||||||
|
|
||||||
image_size_mb = decimal.Decimal(image_stats.max) / units.Mi
|
image_size_mb = decimal.Decimal(image_stats.max) / units.Mi
|
||||||
image_data.append(
|
|
||||||
self.t_cloudkitty.format_item(image, self.units_mappings[
|
try:
|
||||||
"image"], image_size_mb))
|
image_data.append(self.t_cloudkitty.format_item(
|
||||||
|
image,
|
||||||
|
METRICS_CONF['services_units']['image'],
|
||||||
|
image_size_mb,
|
||||||
|
))
|
||||||
|
# NOTE(mc): deprecated except part kept for backward compatibility.
|
||||||
|
except KeyError:
|
||||||
|
LOG.warning('Error when trying to use yaml metrology conf.')
|
||||||
|
LOG.warning('Fallback on the deprecated oslo config method.')
|
||||||
|
image_data.append(self.t_cloudkitty.format_item(
|
||||||
|
image,
|
||||||
|
self.units_mappings['image'],
|
||||||
|
image_size_mb,
|
||||||
|
))
|
||||||
|
|
||||||
if not image_data:
|
if not image_data:
|
||||||
raise collector.NoDataCollected(self.collector_name, 'image')
|
raise collector.NoDataCollected(self.collector_name, 'image')
|
||||||
@ -255,9 +294,23 @@ class CeilometerCollector(collector.BaseCollector):
|
|||||||
volume)
|
volume)
|
||||||
volume = self._cacher.get_resource_detail('volume',
|
volume = self._cacher.get_resource_detail('volume',
|
||||||
volume_id)
|
volume_id)
|
||||||
volume_data.append(
|
|
||||||
self.t_cloudkitty.format_item(volume, self.units_mappings[
|
try:
|
||||||
"volume"], volume_stats.max))
|
volume_data.append(self.t_cloudkitty.format_item(
|
||||||
|
volume,
|
||||||
|
METRICS_CONF['services_units']['volume'],
|
||||||
|
volume_stats.max,
|
||||||
|
))
|
||||||
|
# NOTE(mc): deprecated except part kept for backward compatibility.
|
||||||
|
except KeyError:
|
||||||
|
LOG.warning('Error when trying to use yaml metrology conf.')
|
||||||
|
LOG.warning('Fallback on the deprecated oslo config method.')
|
||||||
|
volume_data.append(self.t_cloudkitty.format_item(
|
||||||
|
volume,
|
||||||
|
self.units_mappings['volume'],
|
||||||
|
volume_stats.max,
|
||||||
|
))
|
||||||
|
|
||||||
if not volume_data:
|
if not volume_data:
|
||||||
raise collector.NoDataCollected(self.collector_name, 'volume')
|
raise collector.NoDataCollected(self.collector_name, 'volume')
|
||||||
return self.t_cloudkitty.format_service('volume', volume_data)
|
return self.t_cloudkitty.format_service('volume', volume_data)
|
||||||
@ -294,9 +347,22 @@ class CeilometerCollector(collector.BaseCollector):
|
|||||||
tap_id)
|
tap_id)
|
||||||
|
|
||||||
tap_bw_mb = decimal.Decimal(tap_stat.max) / units.M
|
tap_bw_mb = decimal.Decimal(tap_stat.max) / units.M
|
||||||
bw_data.append(
|
|
||||||
self.t_cloudkitty.format_item(tap, self.units_mappings[
|
try:
|
||||||
"network.bw." + direction], tap_bw_mb))
|
bw_data.append(self.t_cloudkitty.format_item(
|
||||||
|
tap,
|
||||||
|
METRICS_CONF['services_units']['network.bw.' + direction],
|
||||||
|
tap_bw_mb,
|
||||||
|
))
|
||||||
|
# NOTE(mc): deprecated except part kept for backward compatibility.
|
||||||
|
except KeyError:
|
||||||
|
LOG.warning('Error when trying to use yaml metrology conf.')
|
||||||
|
LOG.warning('Fallback on the deprecated oslo config method.')
|
||||||
|
bw_data.append(self.t_cloudkitty.format_item(
|
||||||
|
tap,
|
||||||
|
self.units_mappings['network.bw.' + direction],
|
||||||
|
tap_bw_mb,
|
||||||
|
))
|
||||||
|
|
||||||
ck_res_name = 'network.bw.{}'.format(direction)
|
ck_res_name = 'network.bw.{}'.format(direction)
|
||||||
if not bw_data:
|
if not bw_data:
|
||||||
@ -342,9 +408,23 @@ class CeilometerCollector(collector.BaseCollector):
|
|||||||
floating)
|
floating)
|
||||||
floating = self._cacher.get_resource_detail('network.floating',
|
floating = self._cacher.get_resource_detail('network.floating',
|
||||||
floating_id)
|
floating_id)
|
||||||
floating_data.append(
|
|
||||||
self.t_cloudkitty.format_item(floating, self.units_mappings[
|
try:
|
||||||
"network.floating"], 1))
|
floating_data.append(self.t_cloudkitty.format_item(
|
||||||
|
floating,
|
||||||
|
METRICS_CONF['services_units']['network.floating'],
|
||||||
|
1,
|
||||||
|
))
|
||||||
|
# NOTE(mc): deprecated except part kept for backward compatibility.
|
||||||
|
except KeyError:
|
||||||
|
LOG.warning('Error when trying to use yaml metrology conf.')
|
||||||
|
LOG.warning('Fallback on the deprecated oslo config method.')
|
||||||
|
floating_data.append(self.t_cloudkitty.format_item(
|
||||||
|
floating,
|
||||||
|
self.units_mappings['network.floating'],
|
||||||
|
1,
|
||||||
|
))
|
||||||
|
|
||||||
if not floating_data:
|
if not floating_data:
|
||||||
raise collector.NoDataCollected(self.collector_name,
|
raise collector.NoDataCollected(self.collector_name,
|
||||||
'network.floating')
|
'network.floating')
|
||||||
|
@ -18,11 +18,15 @@ import decimal
|
|||||||
from gnocchiclient import client as gclient
|
from gnocchiclient import client as gclient
|
||||||
from keystoneauth1 import loading as ks_loading
|
from keystoneauth1 import loading as ks_loading
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
|
from oslo_log import log as logging
|
||||||
from oslo_utils import units
|
from oslo_utils import units
|
||||||
|
|
||||||
from cloudkitty import collector
|
from cloudkitty import collector
|
||||||
from cloudkitty import utils as ck_utils
|
from cloudkitty import utils as ck_utils
|
||||||
|
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
GNOCCHI_COLLECTOR_OPTS = 'gnocchi_collector'
|
GNOCCHI_COLLECTOR_OPTS = 'gnocchi_collector'
|
||||||
gnocchi_collector_opts = ks_loading.get_auth_common_conf_options()
|
gnocchi_collector_opts = ks_loading.get_auth_common_conf_options()
|
||||||
|
|
||||||
@ -35,6 +39,8 @@ ks_loading.register_auth_conf_options(
|
|||||||
GNOCCHI_COLLECTOR_OPTS)
|
GNOCCHI_COLLECTOR_OPTS)
|
||||||
CONF = cfg.CONF
|
CONF = cfg.CONF
|
||||||
|
|
||||||
|
METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
|
||||||
|
|
||||||
|
|
||||||
class GnocchiCollector(collector.BaseCollector):
|
class GnocchiCollector(collector.BaseCollector):
|
||||||
collector_name = 'gnocchi'
|
collector_name = 'gnocchi'
|
||||||
@ -102,7 +108,15 @@ class GnocchiCollector(collector.BaseCollector):
|
|||||||
try:
|
try:
|
||||||
info["metadata"].extend(transformers['GnocchiTransformer']
|
info["metadata"].extend(transformers['GnocchiTransformer']
|
||||||
.get_metadata(resource_name))
|
.get_metadata(resource_name))
|
||||||
info["unit"] = cls.units_mappings[resource_name][1]
|
|
||||||
|
try:
|
||||||
|
info["unit"] = METRICS_CONF['services_units'][resource_name][1]
|
||||||
|
# NOTE(mc): deprecated except part kept for backward compatibility.
|
||||||
|
except KeyError:
|
||||||
|
LOG.warning('Error when trying to use yaml metrology conf.')
|
||||||
|
LOG.warning('Fallback on the deprecated oslo config method.')
|
||||||
|
info["unit"] = cls.units_mappings[resource_name][1]
|
||||||
|
|
||||||
except KeyError:
|
except KeyError:
|
||||||
pass
|
pass
|
||||||
return info
|
return info
|
||||||
@ -155,26 +169,50 @@ class GnocchiCollector(collector.BaseCollector):
|
|||||||
self.gen_filter(cop="<=", started_at=end))
|
self.gen_filter(cop="<=", started_at=end))
|
||||||
return time_filter
|
return time_filter
|
||||||
|
|
||||||
|
def _expand(self, metrics, resource, name, aggregate, start, end):
|
||||||
|
try:
|
||||||
|
values = self._conn.metric.get_measures(
|
||||||
|
metric=metrics[name],
|
||||||
|
start=ck_utils.ts2dt(start),
|
||||||
|
stop=ck_utils.ts2dt(end),
|
||||||
|
aggregation=aggregate)
|
||||||
|
# NOTE(sheeprine): Get the list of values for the current
|
||||||
|
# metric and get the first result value.
|
||||||
|
# [point_date, granularity, value]
|
||||||
|
# ["2015-11-24T00:00:00+00:00", 86400.0, 64.0]
|
||||||
|
resource[name] = values[0][2]
|
||||||
|
except IndexError:
|
||||||
|
resource[name] = 0
|
||||||
|
except KeyError:
|
||||||
|
# Skip metrics not found
|
||||||
|
pass
|
||||||
|
|
||||||
def _expand_metrics(self, resources, mappings, start, end):
|
def _expand_metrics(self, resources, mappings, start, end):
|
||||||
for resource in resources:
|
for resource in resources:
|
||||||
metrics = resource.get('metrics', {})
|
metrics = resource.get('metrics', {})
|
||||||
for name, aggregate in mappings:
|
try:
|
||||||
try:
|
for mapping in mappings:
|
||||||
values = self._conn.metric.get_measures(
|
self._expand(
|
||||||
metric=metrics[name],
|
metrics,
|
||||||
start=ck_utils.ts2dt(start),
|
resource,
|
||||||
stop=ck_utils.ts2dt(end),
|
mapping.keys()[0],
|
||||||
aggregation=aggregate)
|
mapping.values()[0],
|
||||||
# NOTE(sheeprine): Get the list of values for the current
|
start,
|
||||||
# metric and get the first result value.
|
end,
|
||||||
# [point_date, granularity, value]
|
)
|
||||||
# ["2015-11-24T00:00:00+00:00", 86400.0, 64.0]
|
# NOTE(mc): deprecated except part kept for backward compatibility.
|
||||||
resource[name] = values[0][2]
|
except AttributeError:
|
||||||
except IndexError:
|
LOG.warning('Error when trying to use yaml metrology conf.')
|
||||||
resource[name] = 0
|
LOG.warning('Fallback on the deprecated oslo config method.')
|
||||||
except KeyError:
|
for name, aggregate in mappings:
|
||||||
# Skip metrics not found
|
self._expand(
|
||||||
pass
|
metrics,
|
||||||
|
resource,
|
||||||
|
name,
|
||||||
|
aggregate,
|
||||||
|
start,
|
||||||
|
end,
|
||||||
|
)
|
||||||
|
|
||||||
def get_resources(self, resource_name, start, end,
|
def get_resources(self, resource_name, start, end,
|
||||||
project_id, q_filter=None):
|
project_id, q_filter=None):
|
||||||
@ -196,7 +234,15 @@ class GnocchiCollector(collector.BaseCollector):
|
|||||||
|
|
||||||
# Translating the resource name if needed
|
# Translating the resource name if needed
|
||||||
query_parameters = self._generate_time_filter(start, end)
|
query_parameters = self._generate_time_filter(start, end)
|
||||||
resource_type = self.retrieve_mappings.get(resource_name)
|
|
||||||
|
try:
|
||||||
|
resource_type = METRICS_CONF['services_objects'].get(resource_name)
|
||||||
|
# NOTE(mc): deprecated except part kept for backward compatibility.
|
||||||
|
except KeyError:
|
||||||
|
LOG.warning('Error when trying to use yaml metrology conf.')
|
||||||
|
LOG.warning('Fallback on the deprecated oslo config method.')
|
||||||
|
resource_type = self.retrieve_mappings.get(resource_name)
|
||||||
|
|
||||||
query_parameters.append(
|
query_parameters.append(
|
||||||
self.gen_filter(cop="=", type=resource_type))
|
self.gen_filter(cop="=", type=resource_type))
|
||||||
query_parameters.append(
|
query_parameters.append(
|
||||||
@ -210,7 +256,18 @@ class GnocchiCollector(collector.BaseCollector):
|
|||||||
|
|
||||||
def resource_info(self, resource_name, start, end, project_id,
|
def resource_info(self, resource_name, start, end, project_id,
|
||||||
q_filter=None):
|
q_filter=None):
|
||||||
qty, unit = self.units_mappings.get(resource_name, self.default_unit)
|
try:
|
||||||
|
qty = METRICS_CONF['services_units'][resource_name].keys()[0]
|
||||||
|
unit = METRICS_CONF['services_units'][resource_name].values()[0]
|
||||||
|
# NOTE(mc): deprecated except part kept for backward compatibility.
|
||||||
|
except KeyError:
|
||||||
|
LOG.warning('Error when trying to use yaml metrology conf.')
|
||||||
|
LOG.warning('Fallback on the deprecated oslo config method.')
|
||||||
|
qty, unit = self.units_mappings.get(
|
||||||
|
resource_name,
|
||||||
|
self.default_unit,
|
||||||
|
)
|
||||||
|
|
||||||
resources = self.get_resources(resource_name, start, end,
|
resources = self.get_resources(resource_name, start, end,
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
q_filter=q_filter)
|
q_filter=q_filter)
|
||||||
@ -218,7 +275,15 @@ class GnocchiCollector(collector.BaseCollector):
|
|||||||
for resource in resources:
|
for resource in resources:
|
||||||
resource_data = self.t_gnocchi.strip_resource_data(
|
resource_data = self.t_gnocchi.strip_resource_data(
|
||||||
resource_name, resource)
|
resource_name, resource)
|
||||||
mappings = self.metrics_mappings[resource_name]
|
|
||||||
|
try:
|
||||||
|
mappings = METRICS_CONF['services_metrics'][resource_name]
|
||||||
|
# NOTE(mc): deprecated except part kept for backward compatibility.
|
||||||
|
except KeyError:
|
||||||
|
LOG.warning('Error when trying to use yaml metrology conf.')
|
||||||
|
LOG.warning('Fallback on the deprecated oslo config method.')
|
||||||
|
mappings = self.metrics_mappings[resource_name]
|
||||||
|
|
||||||
self._expand_metrics([resource_data], mappings, start, end)
|
self._expand_metrics([resource_data], mappings, start, end)
|
||||||
resource_data.pop('metrics', None)
|
resource_data.pop('metrics', None)
|
||||||
# Convert network.bw.in, network.bw.out and image unit to MB
|
# Convert network.bw.in, network.bw.out and image unit to MB
|
||||||
|
@ -21,12 +21,16 @@ from keystoneauth1 import loading as ks_loading
|
|||||||
from keystoneclient.v3 import client as ks_client
|
from keystoneclient.v3 import client as ks_client
|
||||||
from monascaclient import client as mclient
|
from monascaclient import client as mclient
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
|
from oslo_log import log as logging
|
||||||
from oslo_utils import units
|
from oslo_utils import units
|
||||||
|
|
||||||
from cloudkitty import collector
|
from cloudkitty import collector
|
||||||
from cloudkitty import transformer
|
from cloudkitty import transformer
|
||||||
from cloudkitty import utils as ck_utils
|
from cloudkitty import utils as ck_utils
|
||||||
|
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
MONASCA_API_VERSION = '2_0'
|
MONASCA_API_VERSION = '2_0'
|
||||||
COLLECTOR_MONASCA_OPTS = 'collector_monasca'
|
COLLECTOR_MONASCA_OPTS = 'collector_monasca'
|
||||||
collector_monasca_opts = ks_loading.get_auth_common_conf_options()
|
collector_monasca_opts = ks_loading.get_auth_common_conf_options()
|
||||||
@ -40,6 +44,8 @@ ks_loading.register_auth_conf_options(
|
|||||||
COLLECTOR_MONASCA_OPTS)
|
COLLECTOR_MONASCA_OPTS)
|
||||||
CONF = cfg.CONF
|
CONF = cfg.CONF
|
||||||
|
|
||||||
|
METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
|
||||||
|
|
||||||
|
|
||||||
class EndpointNotFound(Exception):
|
class EndpointNotFound(Exception):
|
||||||
"""Exception raised if the Monasca endpoint is not found"""
|
"""Exception raised if the Monasca endpoint is not found"""
|
||||||
@ -57,7 +63,7 @@ class MonascaCollector(collector.BaseCollector):
|
|||||||
'network.bw.in': 'network.incoming.bytes',
|
'network.bw.in': 'network.incoming.bytes',
|
||||||
'network.bw.out': 'network.outgoing.bytes',
|
'network.bw.out': 'network.outgoing.bytes',
|
||||||
}
|
}
|
||||||
metric_mappings = {
|
metrics_mappings = {
|
||||||
'compute': [
|
'compute': [
|
||||||
('cpu', 'max'),
|
('cpu', 'max'),
|
||||||
('vpcus', 'max'),
|
('vpcus', 'max'),
|
||||||
@ -77,7 +83,7 @@ class MonascaCollector(collector.BaseCollector):
|
|||||||
}
|
}
|
||||||
# (qty, unit). qty must be either a metric name, an integer
|
# (qty, unit). qty must be either a metric name, an integer
|
||||||
# or a decimal.Decimal object
|
# or a decimal.Decimal object
|
||||||
unit_mappings = {
|
units_mappings = {
|
||||||
'compute': (1, 'instance'),
|
'compute': (1, 'instance'),
|
||||||
'image': ('image.size', 'MB'),
|
'image': ('image.size', 'MB'),
|
||||||
'volume': ('volume.size', 'GB'),
|
'volume': ('volume.size', 'GB'),
|
||||||
@ -127,9 +133,16 @@ class MonascaCollector(collector.BaseCollector):
|
|||||||
def _get_metadata(self, resource_type, transformers):
|
def _get_metadata(self, resource_type, transformers):
|
||||||
info = {}
|
info = {}
|
||||||
try:
|
try:
|
||||||
info['unit'] = self.unit_mappings[resource_type][1]
|
info['unit'] = METRICS_CONF['services_units']
|
||||||
except (KeyError, IndexError):
|
# NOTE(mc): deprecated second try kept for backward compatibility.
|
||||||
info['unit'] = self.default_unit[1]
|
except KeyError:
|
||||||
|
LOG.warning('Error when trying to use yaml metrology conf.')
|
||||||
|
LOG.warning('Fallback on the deprecated oslo config method.')
|
||||||
|
try:
|
||||||
|
info['unit'] = self.units_mappings[resource_type][1]
|
||||||
|
except (KeyError, IndexError):
|
||||||
|
info['unit'] = self.default_unit[1]
|
||||||
|
|
||||||
start = ck_utils.dt2ts(ck_utils.get_month_start())
|
start = ck_utils.dt2ts(ck_utils.get_month_start())
|
||||||
end = ck_utils.dt2ts(ck_utils.get_month_end())
|
end = ck_utils.dt2ts(ck_utils.get_month_end())
|
||||||
try:
|
try:
|
||||||
@ -140,11 +153,19 @@ class MonascaCollector(collector.BaseCollector):
|
|||||||
metadata = self._get_resource_metadata(resource_type, start,
|
metadata = self._get_resource_metadata(resource_type, start,
|
||||||
end, resource_id)
|
end, resource_id)
|
||||||
info['metadata'] = metadata.keys()
|
info['metadata'] = metadata.keys()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
for metric, statistics in self.metric_mappings[resource_type]:
|
for metric, statistics in METRICS_CONF['services_metrics']:
|
||||||
info['metadata'].append(metric)
|
info['metadata'].append(metric)
|
||||||
except (KeyError, IndexError):
|
# NOTE(mc): deprecated second try kept for backward compatibility.
|
||||||
pass
|
except KeyError:
|
||||||
|
LOG.warning('Error when trying to use yaml metrology conf.')
|
||||||
|
LOG.warning('Fallback on the deprecated oslo config method.')
|
||||||
|
try:
|
||||||
|
for metric, statistics in self.metrics_mappings[resource_type]:
|
||||||
|
info['metadata'].append(metric)
|
||||||
|
except (KeyError, IndexError):
|
||||||
|
pass
|
||||||
return info
|
return info
|
||||||
|
|
||||||
# NOTE(lukapeschke) if anyone sees a better way to do this,
|
# NOTE(lukapeschke) if anyone sees a better way to do this,
|
||||||
@ -158,7 +179,14 @@ class MonascaCollector(collector.BaseCollector):
|
|||||||
return tmp._get_metadata(resource_type, transformers)
|
return tmp._get_metadata(resource_type, transformers)
|
||||||
|
|
||||||
def _get_resource_metadata(self, resource_type, start, end, resource_id):
|
def _get_resource_metadata(self, resource_type, start, end, resource_id):
|
||||||
meter = self.retrieve_mappings.get(resource_type)
|
try:
|
||||||
|
meter = METRICS_CONF['services_objects'].get(resource_type)
|
||||||
|
# NOTE(mc): deprecated except part kept for backward compatibility.
|
||||||
|
except KeyError:
|
||||||
|
LOG.warning('Error when trying to use yaml metrology conf.')
|
||||||
|
LOG.warning('Fallback on the deprecated oslo config method.')
|
||||||
|
meter = self.retrieve_mappings.get(resource_type)
|
||||||
|
|
||||||
if not meter:
|
if not meter:
|
||||||
return {}
|
return {}
|
||||||
measurements = self._conn.metrics.list_measurements(
|
measurements = self._conn.metrics.list_measurements(
|
||||||
@ -209,7 +237,14 @@ class MonascaCollector(collector.BaseCollector):
|
|||||||
|
|
||||||
def active_resources(self, resource_type, start,
|
def active_resources(self, resource_type, start,
|
||||||
end, project_id, **kwargs):
|
end, project_id, **kwargs):
|
||||||
meter = self.retrieve_mappings.get(resource_type)
|
try:
|
||||||
|
meter = METRICS_CONF['services_objects'].get(resource_type)
|
||||||
|
# NOTE(mc): deprecated except part kept for backward compatibility.
|
||||||
|
except KeyError:
|
||||||
|
LOG.warning('Error when trying to use yaml metrology conf.')
|
||||||
|
LOG.warning('Fallback on the deprecated oslo config method.')
|
||||||
|
meter = self.retrieve_mappings.get(resource_type)
|
||||||
|
|
||||||
if not meter:
|
if not meter:
|
||||||
return {}
|
return {}
|
||||||
dimensions = {}
|
dimensions = {}
|
||||||
@ -245,7 +280,21 @@ class MonascaCollector(collector.BaseCollector):
|
|||||||
|
|
||||||
def resource_info(self, resource_type, start, end,
|
def resource_info(self, resource_type, start, end,
|
||||||
project_id, q_filter=None):
|
project_id, q_filter=None):
|
||||||
qty, unit = self.unit_mappings.get(resource_type, self.default_unit)
|
|
||||||
|
try:
|
||||||
|
qty, unit = METRICS_CONF['services_units'].get(
|
||||||
|
resource_type,
|
||||||
|
self.default_unit
|
||||||
|
)
|
||||||
|
# NOTE(mc): deprecated except part kept for backward compatibility.
|
||||||
|
except KeyError:
|
||||||
|
LOG.warning('Error when trying to use yaml metrology conf.')
|
||||||
|
LOG.warning('Fallback on the deprecated oslo config method.')
|
||||||
|
qty, unit = self.units_mappings.get(
|
||||||
|
resource_type,
|
||||||
|
self.default_unit
|
||||||
|
)
|
||||||
|
|
||||||
active_resource_ids = self.active_resources(
|
active_resource_ids = self.active_resources(
|
||||||
resource_type, start, end, project_id
|
resource_type, start, end, project_id
|
||||||
)
|
)
|
||||||
@ -253,11 +302,27 @@ class MonascaCollector(collector.BaseCollector):
|
|||||||
for resource_id in active_resource_ids:
|
for resource_id in active_resource_ids:
|
||||||
data = self._get_resource_metadata(resource_type, start,
|
data = self._get_resource_metadata(resource_type, start,
|
||||||
end, resource_id)
|
end, resource_id)
|
||||||
mappings = self.metric_mappings[resource_type]
|
|
||||||
|
try:
|
||||||
|
mappings = METRICS_CONF['services_metrics'][resource_type]
|
||||||
|
# NOTE(mc): deprecated except part kept for backward compatibility.
|
||||||
|
except KeyError:
|
||||||
|
LOG.warning('Error when trying to use yaml metrology conf.')
|
||||||
|
LOG.warning('Fallback on the deprecated oslo config method.')
|
||||||
|
mappings = self.metrics_mappings[resource_type]
|
||||||
|
|
||||||
self._expand_metrics(data, resource_id, mappings, start, end)
|
self._expand_metrics(data, resource_id, mappings, start, end)
|
||||||
resource_qty = qty
|
resource_qty = qty
|
||||||
if not (isinstance(qty, int) or isinstance(qty, decimal.Decimal)):
|
if not (isinstance(qty, int) or isinstance(qty, decimal.Decimal)):
|
||||||
resource_qty = data[self.retrieve_mappings[resource_type]]
|
try:
|
||||||
|
resource_qty = METRICS_CONF['services_objects']
|
||||||
|
# NOTE(mc): deprecated except part kept for backward compat.
|
||||||
|
except KeyError:
|
||||||
|
LOG.warning('Error when trying to use yaml metrology conf')
|
||||||
|
msg = 'Fallback on the deprecated oslo config method'
|
||||||
|
LOG.warning(msg)
|
||||||
|
resource_qty = data[self.retrieve_mappings[resource_type]]
|
||||||
|
|
||||||
resource = self.t_cloudkitty.format_item(data, unit, resource_qty)
|
resource = self.t_cloudkitty.format_item(data, unit, resource_qty)
|
||||||
resource['desc']['resource_id'] = resource_id
|
resource['desc']['resource_id'] = resource_id
|
||||||
resource['resource_id'] = resource_id
|
resource['resource_id'] = resource_id
|
||||||
|
@ -17,7 +17,6 @@ import copy
|
|||||||
import itertools
|
import itertools
|
||||||
|
|
||||||
import cloudkitty.api.app
|
import cloudkitty.api.app
|
||||||
import cloudkitty.collector
|
|
||||||
import cloudkitty.collector.ceilometer
|
import cloudkitty.collector.ceilometer
|
||||||
import cloudkitty.collector.gnocchi
|
import cloudkitty.collector.gnocchi
|
||||||
import cloudkitty.collector.monasca
|
import cloudkitty.collector.monasca
|
||||||
@ -28,6 +27,7 @@ import cloudkitty.storage
|
|||||||
import cloudkitty.storage.gnocchi
|
import cloudkitty.storage.gnocchi
|
||||||
import cloudkitty.tenant_fetcher
|
import cloudkitty.tenant_fetcher
|
||||||
import cloudkitty.tenant_fetcher.keystone
|
import cloudkitty.tenant_fetcher.keystone
|
||||||
|
import cloudkitty.utils
|
||||||
|
|
||||||
__all__ = ['list_opts']
|
__all__ = ['list_opts']
|
||||||
|
|
||||||
@ -35,7 +35,7 @@ _opts = [
|
|||||||
('api', list(itertools.chain(
|
('api', list(itertools.chain(
|
||||||
cloudkitty.api.app.api_opts,))),
|
cloudkitty.api.app.api_opts,))),
|
||||||
('collect', list(itertools.chain(
|
('collect', list(itertools.chain(
|
||||||
cloudkitty.collector.collect_opts))),
|
cloudkitty.utils.collect_opts))),
|
||||||
('ceilometer_collector', list(itertools.chain(
|
('ceilometer_collector', list(itertools.chain(
|
||||||
cloudkitty.collector.ceilometer.ceilometer_collector_opts))),
|
cloudkitty.collector.ceilometer.ceilometer_collector_opts))),
|
||||||
('collector_monasca', list(itertools.chain(
|
('collector_monasca', list(itertools.chain(
|
||||||
|
@ -51,6 +51,8 @@ orchestrator_opts = [
|
|||||||
]
|
]
|
||||||
CONF.register_opts(orchestrator_opts, group='orchestrator')
|
CONF.register_opts(orchestrator_opts, group='orchestrator')
|
||||||
|
|
||||||
|
METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
|
||||||
|
|
||||||
FETCHERS_NAMESPACE = 'cloudkitty.tenant.fetchers'
|
FETCHERS_NAMESPACE = 'cloudkitty.tenant.fetchers'
|
||||||
PROCESSORS_NAMESPACE = 'cloudkitty.rating.processors'
|
PROCESSORS_NAMESPACE = 'cloudkitty.rating.processors'
|
||||||
|
|
||||||
@ -152,8 +154,8 @@ class Worker(BaseWorker):
|
|||||||
def __init__(self, collector, storage, tenant_id=None):
|
def __init__(self, collector, storage, tenant_id=None):
|
||||||
self._collector = collector
|
self._collector = collector
|
||||||
self._storage = storage
|
self._storage = storage
|
||||||
self._period = CONF.collect.period
|
self._period = METRICS_CONF['period']
|
||||||
self._wait_time = CONF.collect.wait_periods * self._period
|
self._wait_time = METRICS_CONF['wait_periods'] * self._period
|
||||||
|
|
||||||
super(Worker, self).__init__(tenant_id)
|
super(Worker, self).__init__(tenant_id)
|
||||||
|
|
||||||
@ -180,7 +182,7 @@ class Worker(BaseWorker):
|
|||||||
if not timestamp:
|
if not timestamp:
|
||||||
break
|
break
|
||||||
|
|
||||||
for service in CONF.collect.services:
|
for service in METRICS_CONF['services']:
|
||||||
try:
|
try:
|
||||||
try:
|
try:
|
||||||
data = self._collect(service, timestamp)
|
data = self._collect(service, timestamp)
|
||||||
@ -232,8 +234,8 @@ class Orchestrator(object):
|
|||||||
uuidutils.generate_uuid().encode('ascii'))
|
uuidutils.generate_uuid().encode('ascii'))
|
||||||
self.coord.start()
|
self.coord.start()
|
||||||
|
|
||||||
self._period = CONF.collect.period
|
self._period = METRICS_CONF['period']
|
||||||
self._wait_time = CONF.collect.wait_periods * self._period
|
self._wait_time = METRICS_CONF['wait_periods'] * self._period
|
||||||
|
|
||||||
def _lock(self, tenant_id):
|
def _lock(self, tenant_id):
|
||||||
lock_name = b"cloudkitty-" + str(tenant_id).encode('ascii')
|
lock_name = b"cloudkitty-" + str(tenant_id).encode('ascii')
|
||||||
|
@ -33,12 +33,14 @@ storage_opts = [
|
|||||||
CONF = cfg.CONF
|
CONF = cfg.CONF
|
||||||
CONF.register_opts(storage_opts, group='storage')
|
CONF.register_opts(storage_opts, group='storage')
|
||||||
|
|
||||||
|
METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
|
||||||
|
|
||||||
STORAGES_NAMESPACE = 'cloudkitty.storage.backends'
|
STORAGES_NAMESPACE = 'cloudkitty.storage.backends'
|
||||||
|
|
||||||
|
|
||||||
def get_storage(collector=None):
|
def get_storage(collector=None):
|
||||||
storage_args = {
|
storage_args = {
|
||||||
'period': CONF.collect.period,
|
'period': METRICS_CONF['period'],
|
||||||
'collector': collector if collector else ck_collector.get_collector()}
|
'collector': collector if collector else ck_collector.get_collector()}
|
||||||
backend = driver.DriverManager(
|
backend = driver.DriverManager(
|
||||||
STORAGES_NAMESPACE,
|
STORAGES_NAMESPACE,
|
||||||
@ -63,7 +65,7 @@ class BaseStorage(object):
|
|||||||
Handle incoming data from the global orchestrator, and store them.
|
Handle incoming data from the global orchestrator, and store them.
|
||||||
"""
|
"""
|
||||||
def __init__(self, **kwargs):
|
def __init__(self, **kwargs):
|
||||||
self._period = kwargs.get('period', CONF.collect.period)
|
self._period = kwargs.get('period', METRICS_CONF['period'])
|
||||||
self._collector = kwargs.get('collector')
|
self._collector = kwargs.get('collector')
|
||||||
|
|
||||||
# State vars
|
# State vars
|
||||||
|
@ -34,7 +34,7 @@ from cloudkitty import utils as ck_utils
|
|||||||
LOG = log.getLogger(__name__)
|
LOG = log.getLogger(__name__)
|
||||||
CONF = cfg.CONF
|
CONF = cfg.CONF
|
||||||
|
|
||||||
CONF.import_opt('period', 'cloudkitty.collector', 'collect')
|
METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
|
||||||
|
|
||||||
GNOCCHI_STORAGE_OPTS = 'storage_gnocchi'
|
GNOCCHI_STORAGE_OPTS = 'storage_gnocchi'
|
||||||
gnocchi_storage_opts = [
|
gnocchi_storage_opts = [
|
||||||
@ -44,7 +44,7 @@ gnocchi_storage_opts = [
|
|||||||
# The archive policy definition MUST include the collect period granularity
|
# The archive policy definition MUST include the collect period granularity
|
||||||
cfg.StrOpt('archive_policy_definition',
|
cfg.StrOpt('archive_policy_definition',
|
||||||
default='[{"granularity": '
|
default='[{"granularity": '
|
||||||
+ six.text_type(CONF.collect.period) +
|
+ six.text_type(METRICS_CONF['period']) +
|
||||||
', "timespan": "90 days"}, '
|
', "timespan": "90 days"}, '
|
||||||
'{"granularity": 86400, "timespan": "360 days"}, '
|
'{"granularity": 86400, "timespan": "360 days"}, '
|
||||||
'{"granularity": 2592000, "timespan": "1800 days"}]',
|
'{"granularity": 2592000, "timespan": "1800 days"}]',
|
||||||
@ -84,7 +84,7 @@ class GnocchiStorage(storage.BaseStorage):
|
|||||||
CONF.storage_gnocchi.archive_policy_name)
|
CONF.storage_gnocchi.archive_policy_name)
|
||||||
self._archive_policy_definition = json.loads(
|
self._archive_policy_definition = json.loads(
|
||||||
CONF.storage_gnocchi.archive_policy_definition)
|
CONF.storage_gnocchi.archive_policy_definition)
|
||||||
self._period = CONF.collect.period
|
self._period = METRICS_CONF['period']
|
||||||
if "period" in kwargs:
|
if "period" in kwargs:
|
||||||
self._period = kwargs["period"]
|
self._period = kwargs["period"]
|
||||||
|
|
||||||
|
@ -8,12 +8,12 @@ tests:
|
|||||||
response_json_paths:
|
response_json_paths:
|
||||||
$.collect.services.`len`: 6
|
$.collect.services.`len`: 6
|
||||||
$.collect.services[0]: compute
|
$.collect.services[0]: compute
|
||||||
$.collect.services[1]: image
|
$.collect.services[1]: volume
|
||||||
$.collect.services[2]: volume
|
$.collect.services[2]: network.bw.in
|
||||||
$.collect.services[3]: network.bw.in
|
$.collect.services[3]: network.bw.out
|
||||||
$.collect.services[4]: network.bw.out
|
$.collect.services[4]: network.floating
|
||||||
$.collect.services[5]: network.floating
|
$.collect.services[5]: image
|
||||||
$.collect.collector: ceilometer
|
$.collect.collector: gnocchi
|
||||||
$.collect.window: 1800
|
$.collect.window: 1800
|
||||||
$.collect.wait_periods: 2
|
$.collect.wait_periods: 2
|
||||||
$.collect.period: 3600
|
$.collect.period: 3600
|
||||||
@ -42,4 +42,4 @@ tests:
|
|||||||
response_json_paths:
|
response_json_paths:
|
||||||
$.service_id: compute
|
$.service_id: compute
|
||||||
$.unit: instance
|
$.unit: instance
|
||||||
$.metadata.`len`: 10
|
$.metadata.`len`: 8
|
||||||
|
@ -24,7 +24,10 @@ to ease maintenance in case of library modifications.
|
|||||||
import calendar
|
import calendar
|
||||||
import datetime
|
import datetime
|
||||||
import sys
|
import sys
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
from oslo_config import cfg
|
||||||
|
from oslo_log import log as logging
|
||||||
from oslo_utils import timeutils
|
from oslo_utils import timeutils
|
||||||
from six import moves
|
from six import moves
|
||||||
from stevedore import extension
|
from stevedore import extension
|
||||||
@ -33,6 +36,43 @@ from stevedore import extension
|
|||||||
_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f'
|
_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f'
|
||||||
_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
|
_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
collect_opts = [
|
||||||
|
cfg.StrOpt('collector',
|
||||||
|
default='gnocchi',
|
||||||
|
deprecated_for_removal=True,
|
||||||
|
help='Data collector.'),
|
||||||
|
cfg.IntOpt('window',
|
||||||
|
default=1800,
|
||||||
|
deprecated_for_removal=True,
|
||||||
|
help='Number of samples to collect per call.'),
|
||||||
|
cfg.IntOpt('period',
|
||||||
|
default=3600,
|
||||||
|
deprecated_for_removal=True,
|
||||||
|
help='Rating period in seconds.'),
|
||||||
|
cfg.IntOpt('wait_periods',
|
||||||
|
default=2,
|
||||||
|
deprecated_for_removal=True,
|
||||||
|
help='Wait for N periods before collecting new data.'),
|
||||||
|
cfg.ListOpt('services',
|
||||||
|
default=[
|
||||||
|
'compute',
|
||||||
|
'volume',
|
||||||
|
'network.bw.in',
|
||||||
|
'network.bw.out',
|
||||||
|
'network.floating',
|
||||||
|
'image',
|
||||||
|
],
|
||||||
|
deprecated_for_removal=True,
|
||||||
|
help='Services to monitor.'),
|
||||||
|
cfg.StrOpt('metrics_conf',
|
||||||
|
default='/etc/cloudkitty/metrics.yml',
|
||||||
|
help='Metrology configuration file.'),
|
||||||
|
]
|
||||||
|
CONF = cfg.CONF
|
||||||
|
CONF.register_opts(collect_opts, 'collect')
|
||||||
|
|
||||||
|
|
||||||
def isotime(at=None, subsecond=False):
|
def isotime(at=None, subsecond=False):
|
||||||
"""Stringify time in ISO 8601 format."""
|
"""Stringify time in ISO 8601 format."""
|
||||||
@ -190,3 +230,30 @@ def check_time_state(timestamp=None, period=0, wait_time=0):
|
|||||||
if next_timestamp + wait_time < now:
|
if next_timestamp + wait_time < now:
|
||||||
return next_timestamp
|
return next_timestamp
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def get_metrics_conf(conf_path):
|
||||||
|
"""Return loaded yaml metrology configuration.
|
||||||
|
|
||||||
|
In case of empty /etc/cloudkitty folder,
|
||||||
|
a fallback is done on the former deprecated oslo config method.
|
||||||
|
"""
|
||||||
|
res = None
|
||||||
|
try:
|
||||||
|
with open(conf_path) as conf:
|
||||||
|
res = yaml.load(conf)
|
||||||
|
res = res[0]
|
||||||
|
except Exception as exc:
|
||||||
|
LOG.warning('Error when trying to retrieve yaml metrology conf file.')
|
||||||
|
LOG.warning(exc)
|
||||||
|
LOG.warning('Fallback on the deprecated oslo config method.')
|
||||||
|
|
||||||
|
try:
|
||||||
|
res = {key: val for key, val in CONF.collect.items()}
|
||||||
|
except Exception as exc:
|
||||||
|
err_msg = 'Error when trying to retrieve ' \
|
||||||
|
'deprecated oslo config method.'
|
||||||
|
LOG.error(err_msg)
|
||||||
|
LOG.error(exc)
|
||||||
|
|
||||||
|
return res
|
||||||
|
@ -134,6 +134,7 @@ function configure_cloudkitty {
|
|||||||
|
|
||||||
cp $CLOUDKITTY_DIR$CLOUDKITTY_CONF_DIR/policy.json $CLOUDKITTY_CONF_DIR
|
cp $CLOUDKITTY_DIR$CLOUDKITTY_CONF_DIR/policy.json $CLOUDKITTY_CONF_DIR
|
||||||
cp $CLOUDKITTY_DIR$CLOUDKITTY_CONF_DIR/api_paste.ini $CLOUDKITTY_CONF_DIR
|
cp $CLOUDKITTY_DIR$CLOUDKITTY_CONF_DIR/api_paste.ini $CLOUDKITTY_CONF_DIR
|
||||||
|
cp $CLOUDKITTY_DIR$CLOUDKITTY_CONF_DIR/metrics.yml $CLOUDKITTY_CONF_DIR
|
||||||
iniset_rpc_backend cloudkitty $CLOUDKITTY_CONF DEFAULT
|
iniset_rpc_backend cloudkitty $CLOUDKITTY_CONF DEFAULT
|
||||||
|
|
||||||
iniset $CLOUDKITTY_CONF DEFAULT notification_topics 'notifications'
|
iniset $CLOUDKITTY_CONF DEFAULT notification_topics 'notifications'
|
||||||
@ -161,6 +162,7 @@ function configure_cloudkitty {
|
|||||||
iniset $CLOUDKITTY_CONF collect collector $CLOUDKITTY_COLLECTOR
|
iniset $CLOUDKITTY_CONF collect collector $CLOUDKITTY_COLLECTOR
|
||||||
iniset $CLOUDKITTY_CONF ${CLOUDKITTY_COLLECTOR}_collector auth_section authinfos
|
iniset $CLOUDKITTY_CONF ${CLOUDKITTY_COLLECTOR}_collector auth_section authinfos
|
||||||
iniset $CLOUDKITTY_CONF collect services $CLOUDKITTY_SERVICES
|
iniset $CLOUDKITTY_CONF collect services $CLOUDKITTY_SERVICES
|
||||||
|
iniset $CLOUDKITTY_CONF collect metrics_conf $CLOUDKITTY_CONF_DIR/$CLOUDKITTY_METRICS_CONF
|
||||||
|
|
||||||
# output
|
# output
|
||||||
iniset $CLOUDKITTY_CONF output backend $CLOUDKITTY_OUTPUT_BACKEND
|
iniset $CLOUDKITTY_CONF output backend $CLOUDKITTY_OUTPUT_BACKEND
|
||||||
|
@ -43,6 +43,7 @@ CLOUDKITTY_PRICING_TENANT=${CLOUDKITTY_PRICING_TENANT:-"demo"}
|
|||||||
# Set CloudKitty collect info
|
# Set CloudKitty collect info
|
||||||
CLOUDKITTY_COLLECTOR=${CLOUDKITTY_COLLECTOR:-ceilometer}
|
CLOUDKITTY_COLLECTOR=${CLOUDKITTY_COLLECTOR:-ceilometer}
|
||||||
CLOUDKITTY_SERVICES=${CLOUDKITTY_SERVICES:-compute}
|
CLOUDKITTY_SERVICES=${CLOUDKITTY_SERVICES:-compute}
|
||||||
|
CLOUDKITTY_METRICS_CONF=metrics.yml
|
||||||
|
|
||||||
# Set CloudKitty output info
|
# Set CloudKitty output info
|
||||||
CLOUDKITTY_OUTPUT_BACKEND=${CLOUDKITTY_OUTPUT_BACKEND:-"cloudkitty.backend.file.FileBackend"}
|
CLOUDKITTY_OUTPUT_BACKEND=${CLOUDKITTY_OUTPUT_BACKEND:-"cloudkitty.backend.file.FileBackend"}
|
||||||
|
@ -167,18 +167,83 @@ Three collectors are available: Ceilometer (deprecated, see the Telemetry
|
|||||||
documentation), Gnocchi and Monasca. The Monasca collector collects metrics
|
documentation), Gnocchi and Monasca. The Monasca collector collects metrics
|
||||||
published by the Ceilometer agent to Monasca using Ceilosca_.
|
published by the Ceilometer agent to Monasca using Ceilosca_.
|
||||||
|
|
||||||
|
The collect information, is separated from the Cloudkitty configuration file, in a yaml one.
|
||||||
|
|
||||||
|
This allows Cloudkitty users to change metrology configuration,
|
||||||
|
without modifying source code or Cloudkitty configuration file.
|
||||||
|
|
||||||
.. code-block:: ini
|
.. code-block:: ini
|
||||||
|
|
||||||
[collect]
|
[collect]
|
||||||
collector = gnocchi
|
metrics_conf = /etc/cloudkitty/metrics.yml
|
||||||
# Metrics are collected every 3600 seconds
|
|
||||||
period = 3600
|
|
||||||
# By default, only the compute service is enabled
|
|
||||||
services = compute, volume, network.bw.in, network.bw.out, network.floating, image
|
|
||||||
|
|
||||||
[gnocchi_collector]
|
[gnocchi_collector]
|
||||||
auth_section = ks_auth
|
auth_section = ks_auth
|
||||||
|
|
||||||
|
The ``/etc/cloudkitty/metrics.yml`` file looks like this:
|
||||||
|
|
||||||
|
.. code-block:: yaml
|
||||||
|
|
||||||
|
- name: OpenStack
|
||||||
|
|
||||||
|
collector: gnocchi
|
||||||
|
period: 3600
|
||||||
|
wait_period: 2
|
||||||
|
window: 1800
|
||||||
|
|
||||||
|
services:
|
||||||
|
- compute
|
||||||
|
- volume
|
||||||
|
- network.bw.in
|
||||||
|
- network.bw.out
|
||||||
|
- network.floating
|
||||||
|
- image
|
||||||
|
|
||||||
|
services_objects:
|
||||||
|
compute: instance
|
||||||
|
volume: volume
|
||||||
|
network.bw.in: instance_network_interface
|
||||||
|
network.bw.out: instance_network_interface
|
||||||
|
network.floating: network
|
||||||
|
image: image
|
||||||
|
|
||||||
|
services_metrics:
|
||||||
|
compute:
|
||||||
|
- vcpus: max
|
||||||
|
- memory: max
|
||||||
|
- cpu: max
|
||||||
|
- disk.root.size: max
|
||||||
|
- disk.ephemeral.size: max
|
||||||
|
volume:
|
||||||
|
- volume.size: max
|
||||||
|
network.bw.in:
|
||||||
|
- network.incoming.bytes: max
|
||||||
|
network.bw.out:
|
||||||
|
- network.outgoing.bytes: max
|
||||||
|
network.floating:
|
||||||
|
- ip.floating: max
|
||||||
|
image:
|
||||||
|
- image.size: max
|
||||||
|
- image.download: max
|
||||||
|
- image.serve: max
|
||||||
|
|
||||||
|
services_units:
|
||||||
|
compute:
|
||||||
|
1: instance
|
||||||
|
volume:
|
||||||
|
volume.size: GB
|
||||||
|
network.bw.in:
|
||||||
|
network.incoming.bytes: MB
|
||||||
|
network.bw.out:
|
||||||
|
network.outgoing.bytes: MB
|
||||||
|
network.floating:
|
||||||
|
1: ip
|
||||||
|
image:
|
||||||
|
image.size: MB
|
||||||
|
default_unit:
|
||||||
|
1: unknown
|
||||||
|
|
||||||
|
|
||||||
Setup the database and storage backend
|
Setup the database and storage backend
|
||||||
--------------------------------------
|
--------------------------------------
|
||||||
|
|
||||||
@ -274,4 +339,4 @@ Choose and start the API server
|
|||||||
$ cloudkitty-api -p 8889
|
$ cloudkitty-api -p 8889
|
||||||
|
|
||||||
|
|
||||||
.. _Ceilosca: https://github.com/openstack/monasca-ceilometer
|
.. _Ceilosca: https://github.com/openstack/monasca-ceilometer
|
||||||
|
58
etc/cloudkitty/metrics.yml
Normal file
58
etc/cloudkitty/metrics.yml
Normal file
@ -0,0 +1,58 @@
|
|||||||
|
- name: OpenStack
|
||||||
|
|
||||||
|
collector: gnocchi
|
||||||
|
period: 3600
|
||||||
|
wait_periods: 2
|
||||||
|
window: 1800
|
||||||
|
|
||||||
|
services:
|
||||||
|
- compute
|
||||||
|
- volume
|
||||||
|
- network.bw.in
|
||||||
|
- network.bw.out
|
||||||
|
- network.floating
|
||||||
|
- image
|
||||||
|
|
||||||
|
services_objects:
|
||||||
|
compute: instance
|
||||||
|
volume: volume
|
||||||
|
network.bw.out: instance_network_interface
|
||||||
|
network.bw.in: instance_network_interface
|
||||||
|
network.floating: network
|
||||||
|
image: image
|
||||||
|
|
||||||
|
services_metrics:
|
||||||
|
compute:
|
||||||
|
- vcpus: max
|
||||||
|
- memory: max
|
||||||
|
- cpu: max
|
||||||
|
- disk.root.size: max
|
||||||
|
- disk.ephemeral.size: max
|
||||||
|
volume:
|
||||||
|
- volume.size: max
|
||||||
|
network.bw.in:
|
||||||
|
- network.incoming: max
|
||||||
|
network.bw.out:
|
||||||
|
- network.outgoing.bytes: max
|
||||||
|
network.floating:
|
||||||
|
- ip.floating: max
|
||||||
|
image:
|
||||||
|
- image.size: max
|
||||||
|
- image.download: max
|
||||||
|
- image.serve: max
|
||||||
|
|
||||||
|
services_units:
|
||||||
|
compute:
|
||||||
|
1: instance
|
||||||
|
volume:
|
||||||
|
volume.size: GB
|
||||||
|
network.bw.in:
|
||||||
|
network.incoming.bytes: MB
|
||||||
|
network.bw.out:
|
||||||
|
network.outgoing.bytes: MB
|
||||||
|
network.floating:
|
||||||
|
1: ip
|
||||||
|
image:
|
||||||
|
image.size: MB
|
||||||
|
default_unit:
|
||||||
|
1: unknown
|
Loading…
Reference in New Issue
Block a user