Merge "Add a collector for Monasca"
This commit is contained in:
Zuul
Gerrit Code Review
commit
dd6741ad06
273
cloudkitty/collector/monasca.py
Normal file
273
cloudkitty/collector/monasca.py
Normal file
@ -0,0 +1,273 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2017 Objectif Libre
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
# @author: Luka Peschke
|
||||
#
|
||||
import decimal
|
||||
|
||||
from keystoneauth1 import loading as ks_loading
|
||||
from keystoneclient.v3 import client as ks_client
|
||||
from monascaclient import client as mclient
|
||||
from oslo_config import cfg
|
||||
from oslo_utils import units
|
||||
|
||||
from cloudkitty import collector
|
||||
from cloudkitty import transformer
|
||||
from cloudkitty import utils as ck_utils
|
||||
|
||||
MONASCA_API_VERSION = '2_0'
|
||||
COLLECTOR_MONASCA_OPTS = 'collector_monasca'
|
||||
collector_monasca_opts = ks_loading.get_auth_common_conf_options()
|
||||
|
||||
cfg.CONF.register_opts(collector_monasca_opts, COLLECTOR_MONASCA_OPTS)
|
||||
ks_loading.register_session_conf_options(
|
||||
cfg.CONF,
|
||||
COLLECTOR_MONASCA_OPTS)
|
||||
ks_loading.register_auth_conf_options(
|
||||
cfg.CONF,
|
||||
COLLECTOR_MONASCA_OPTS)
|
||||
CONF = cfg.CONF
|
||||
|
||||
|
||||
class EndpointNotFound(Exception):
|
||||
"""Exception raised if the Monasca endpoint is not found"""
|
||||
pass
|
||||
|
||||
|
||||
class MonascaCollector(collector.BaseCollector):
|
||||
collector_name = 'monasca'
|
||||
dependencies = ['CloudKittyFormatTransformer']
|
||||
retrieve_mappings = {
|
||||
'compute': 'cpu',
|
||||
'image': 'image.size',
|
||||
'volume': 'volume.size',
|
||||
'network.floating': 'ip.floating',
|
||||
'network.bw.in': 'network.incoming.bytes',
|
||||
'network.bw.out': 'network.outgoing.bytes',
|
||||
}
|
||||
metric_mappings = {
|
||||
'compute': [
|
||||
('cpu', 'max'),
|
||||
('vpcus', 'max'),
|
||||
('memory', 'max')],
|
||||
'image': [
|
||||
('image.size', 'max'),
|
||||
('image.download', 'max'),
|
||||
('image.serve', 'max')],
|
||||
'volume': [
|
||||
('volume.size', 'max')],
|
||||
'network.bw.in': [
|
||||
('network.incoming.bytes', 'max')],
|
||||
'network.bw.out': [
|
||||
('network.outgoing.bytes', 'max')],
|
||||
'network.floating': [
|
||||
('ip.floating', 'max')],
|
||||
}
|
||||
# (qty, unit). qty must be either a metric name, an integer
|
||||
# or a decimal.Decimal object
|
||||
unit_mappings = {
|
||||
'compute': (1, 'instance'),
|
||||
'image': ('image.size', 'MB'),
|
||||
'volume': ('volume.size', 'GB'),
|
||||
'network.bw.out': ('network.outgoing.bytes', 'MB'),
|
||||
'network.bw.in': ('network.incoming.bytes', 'MB'),
|
||||
'network.floating': (1, 'ip'),
|
||||
}
|
||||
default_unit = (1, 'unknown')
|
||||
|
||||
def __init__(self, transformers, **kwargs):
|
||||
super(MonascaCollector, self).__init__(transformers, **kwargs)
|
||||
|
||||
self.t_cloudkitty = self.transformers['CloudKittyFormatTransformer']
|
||||
|
||||
self.auth = ks_loading.load_auth_from_conf_options(
|
||||
CONF,
|
||||
COLLECTOR_MONASCA_OPTS)
|
||||
self.session = ks_loading.load_session_from_conf_options(
|
||||
CONF,
|
||||
COLLECTOR_MONASCA_OPTS,
|
||||
auth=self.auth)
|
||||
self.ks_client = ks_client.Client(session=self.session)
|
||||
self.mon_endpoint = self._get_monasca_endpoint()
|
||||
if not self.mon_endpoint:
|
||||
raise EndpointNotFound()
|
||||
# NOTE (lukapeschke) session authentication should be possible starting
|
||||
# with OpenStack Q release.
|
||||
self._conn = mclient.Client(
|
||||
api_version=MONASCA_API_VERSION,
|
||||
session=self.session,
|
||||
endpoint=self.mon_endpoint)
|
||||
|
||||
# NOTE(lukapeschke) This function should be removed as soon as the endpoint
|
||||
# it no longer required by monascaclient
|
||||
def _get_monasca_endpoint(self, service_name='monasca',
|
||||
endpoint_interface_type='public'):
|
||||
service_list = self.ks_client.services.list(name=service_name)
|
||||
if not service_list:
|
||||
return None
|
||||
mon_service = service_list[0]
|
||||
endpoints = self.ks_client.endpoints.list(mon_service.id)
|
||||
for endpoint in endpoints:
|
||||
if endpoint.interface == endpoint_interface_type:
|
||||
return endpoint.url
|
||||
return None
|
||||
|
||||
def _get_metadata(self, resource_type, transformers):
|
||||
info = {}
|
||||
try:
|
||||
info['unit'] = self.unit_mappings[resource_type][1]
|
||||
except (KeyError, IndexError):
|
||||
info['unit'] = self.default_unit[1]
|
||||
start = ck_utils.dt2ts(ck_utils.get_month_start())
|
||||
end = ck_utils.dt2ts(ck_utils.get_month_end())
|
||||
try:
|
||||
resource_id = self.active_resources(resource_type, start,
|
||||
end, None)[0]
|
||||
except IndexError:
|
||||
resource_id = ''
|
||||
metadata = self._get_resource_metadata(resource_type, start,
|
||||
end, resource_id)
|
||||
info['metadata'] = metadata.keys()
|
||||
try:
|
||||
for metric, statistics in self.metric_mappings[resource_type]:
|
||||
info['metadata'].append(metric)
|
||||
except (KeyError, IndexError):
|
||||
pass
|
||||
return info
|
||||
|
||||
# NOTE(lukapeschke) if anyone sees a better way to do this,
|
||||
# please make a patch
|
||||
@classmethod
|
||||
def get_metadata(cls, resource_type, transformers):
|
||||
args = {
|
||||
'transformers': transformer.get_transformers(),
|
||||
'period': CONF.collect.period}
|
||||
tmp = cls(**args)
|
||||
return tmp._get_metadata(resource_type, transformers)
|
||||
|
||||
def _get_resource_metadata(self, resource_type, start, end, resource_id):
|
||||
meter = self.retrieve_mappings.get(resource_type)
|
||||
if not meter:
|
||||
return {}
|
||||
measurements = self._conn.metrics.list_measurements(
|
||||
name=meter,
|
||||
start_time=ck_utils.ts2dt(start),
|
||||
end_time=ck_utils.ts2dt(end),
|
||||
merge_metrics=True,
|
||||
dimensions={'resource_id': resource_id},
|
||||
)
|
||||
try:
|
||||
# Getting the last measurement of given period
|
||||
metadata = measurements[-1]['measurements'][-1][2]
|
||||
except (KeyError, IndexError):
|
||||
metadata = {}
|
||||
return metadata
|
||||
|
||||
def _get_resource_qty(self, meter, start, end, resource_id, statistics):
|
||||
# NOTE(lukapeschke) the period trick is used to aggregate
|
||||
# the measurements
|
||||
period = end - start
|
||||
statistics = self._conn.metrics.list_statistics(
|
||||
name=meter,
|
||||
start_time=ck_utils.ts2dt(start),
|
||||
end_time=ck_utils.ts2dt(end),
|
||||
dimensions={'resource_id': resource_id},
|
||||
statistics=statistics,
|
||||
period=period,
|
||||
merge_metrics=True,
|
||||
)
|
||||
try:
|
||||
# If several statistics are returned (should not happen),
|
||||
# use the latest
|
||||
qty = decimal.Decimal(statistics[-1]['statistics'][-1][1])
|
||||
except (KeyError, IndexError):
|
||||
qty = decimal.Decimal(0)
|
||||
return qty
|
||||
|
||||
def _is_resource_active(self, meter, resource_id, start, end):
|
||||
measurements = self._conn.metrics.list_measurements(
|
||||
name=meter,
|
||||
start_time=ck_utils.ts2dt(start),
|
||||
end_time=ck_utils.ts2dt(end),
|
||||
group_by='resource_id',
|
||||
merge_metrics=True,
|
||||
dimensions={'resource_id': resource_id},
|
||||
)
|
||||
return len(measurements) > 0
|
||||
|
||||
def active_resources(self, resource_type, start,
|
||||
end, project_id, **kwargs):
|
||||
meter = self.retrieve_mappings.get(resource_type)
|
||||
if not meter:
|
||||
return {}
|
||||
dimensions = {}
|
||||
dimensions.update(kwargs)
|
||||
if project_id:
|
||||
resources = self._conn.metrics.list(name=meter,
|
||||
tenant_id=project_id,
|
||||
**dimensions)
|
||||
else:
|
||||
resources = self._conn.metrics.list(name=meter,
|
||||
**dimensions)
|
||||
resource_ids = []
|
||||
for resource in resources:
|
||||
try:
|
||||
resource_id = resource['dimensions']['resource_id']
|
||||
if (resource_id not in resource_ids
|
||||
and self._is_resource_active(meter, resource_id,
|
||||
start, end)):
|
||||
resource_ids.append(resource_id)
|
||||
except KeyError:
|
||||
continue
|
||||
return resource_ids
|
||||
|
||||
def _expand_metrics(self, resource, resource_id, mappings, start, end):
|
||||
for name, statistics in mappings:
|
||||
qty = self._get_resource_qty(name, start,
|
||||
end, resource_id, statistics)
|
||||
if name in ['network.outgoing.bytes', 'network.incoming.bytes']:
|
||||
qty = qty / units.M
|
||||
elif 'image.' in name:
|
||||
qty = qty / units.Mi
|
||||
resource[name] = qty
|
||||
|
||||
def resource_info(self, resource_type, start, end,
|
||||
project_id, q_filter=None):
|
||||
qty, unit = self.unit_mappings.get(resource_type, self.default_unit)
|
||||
active_resource_ids = self.active_resources(
|
||||
resource_type, start, end, project_id
|
||||
)
|
||||
resource_data = []
|
||||
for resource_id in active_resource_ids:
|
||||
data = self._get_resource_metadata(resource_type, start,
|
||||
end, resource_id)
|
||||
mappings = self.metric_mappings[resource_type]
|
||||
self._expand_metrics(data, resource_id, mappings, start, end)
|
||||
resource_qty = qty
|
||||
if not (isinstance(qty, int) or isinstance(qty, decimal.Decimal)):
|
||||
resource_qty = data[self.retrieve_mappings[resource_type]]
|
||||
resource = self.t_cloudkitty.format_item(data, unit, resource_qty)
|
||||
resource['desc']['resource_id'] = resource_id
|
||||
resource['resource_id'] = resource_id
|
||||
resource_data.append(resource)
|
||||
return resource_data
|
||||
|
||||
def retrieve(self, resource_type, start, end, project_id, q_filter=None):
|
||||
resources = self.resource_info(resource_type, start, end,
|
||||
project_id=project_id,
|
||||
q_filter=q_filter)
|
||||
if not resources:
|
||||
raise collector.NoDataCollected(self.collector_name, resource_type)
|
||||
return self.t_cloudkitty.format_service(resource_type, resources)
|
||||
@ -20,6 +20,7 @@ import cloudkitty.api.app
|
||||
import cloudkitty.collector
|
||||
import cloudkitty.collector.ceilometer
|
||||
import cloudkitty.collector.gnocchi
|
||||
import cloudkitty.collector.monasca
|
||||
import cloudkitty.config
|
||||
import cloudkitty.orchestrator
|
||||
import cloudkitty.service
|
||||
@ -37,6 +38,8 @@ _opts = [
|
||||
cloudkitty.collector.collect_opts))),
|
||||
('ceilometer_collector', list(itertools.chain(
|
||||
cloudkitty.collector.ceilometer.ceilometer_collector_opts))),
|
||||
('collector_monasca', list(itertools.chain(
|
||||
cloudkitty.collector.monasca.collector_monasca_opts))),
|
||||
('gnocchi_collector', list(itertools.chain(
|
||||
cloudkitty.collector.gnocchi.gnocchi_collector_opts))),
|
||||
('keystone_fetcher', list(itertools.chain(
|
||||
|
||||
@ -163,8 +163,9 @@ If you want to use the pure gnocchi storage, add the following entry:
|
||||
[storage_gnocchi]
|
||||
auth_section = ks_auth
|
||||
|
||||
Two collectors are available: Ceilometer (deprecated, see the Telemetry
|
||||
documentation), and Gnocchi.
|
||||
Three collectors are available: Ceilometer (deprecated, see the Telemetry
|
||||
documentation), Gnocchi and Monasca. The Monasca collector collects metrics
|
||||
published by the Ceilometer agent to Monasca using Ceilosca_.
|
||||
|
||||
.. code-block:: ini
|
||||
|
||||
@ -271,3 +272,6 @@ Choose and start the API server
|
||||
as::
|
||||
|
||||
$ cloudkitty-api -p 8889
|
||||
|
||||
|
||||
.. _Ceilosca: https://github.com/openstack/monasca-ceilometer
|
||||
@ -0,0 +1,5 @@
|
||||
---
|
||||
features:
|
||||
- |
|
||||
A collector for Monasca has been added. It works with telemetry metrics
|
||||
published to Monasca by Ceilometer agent through Ceilosca.
|
||||
@ -7,6 +7,7 @@ eventlet!=0.18.3,>=0.18.2 # MIT
|
||||
keystonemiddleware!=4.1.0,>=4.0.0 # Apache-2.0
|
||||
python-ceilometerclient>=2.2.1 # Apache-2.0
|
||||
gnocchiclient>=2.5.0 # Apache-2.0
|
||||
python-monascaclient>=1.7.0 # Apache-2.0
|
||||
python-keystoneclient!=1.8.0,!=2.1.0,>=1.6.0 # Apache-2.0
|
||||
keystoneauth1>=2.1.0 # Apache-2.0
|
||||
iso8601>=0.1.9 # MIT
|
||||
|
||||
@ -42,6 +42,7 @@ cloudkitty.collector.backends =
|
||||
fake = cloudkitty.collector.fake:CSVCollector
|
||||
ceilometer = cloudkitty.collector.ceilometer:CeilometerCollector
|
||||
gnocchi = cloudkitty.collector.gnocchi:GnocchiCollector
|
||||
monasca = cloudkitty.collector.monasca:MonascaCollector
|
||||
meta = cloudkitty.collector.meta:MetaCollector
|
||||
|
||||
cloudkitty.tenant.fetchers =
|
||||
|
||||
Loading…
Reference in New Issue
Block a user