From cc10eb6e08bf36f6afb729091bfdef613cc4303b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Albert?= Date: Wed, 24 Sep 2014 18:01:39 +0200 Subject: [PATCH] Added support for Multiple Collectors Added a new collector backend: MetaCollector. Added new API endpoints for the MetaCollector configuration. Modified documentation to add new API endpoints. Change-Id: I0216fdb6829fdb2274edf693971c6730727f2cde --- cloudkitty/api/controllers/v1.py | 111 ++++++++++++++++++ cloudkitty/collector/meta.py | 92 +++++++++++++++ cloudkitty/db/api.py | 42 +++++++ ...7dcbd9_added_support_for_meta_collector.py | 30 +++++ cloudkitty/db/sqlalchemy/api.py | 61 ++++++++++ cloudkitty/db/sqlalchemy/models.py | 19 +++ doc/source/webapi/v1.rst | 13 ++ setup.cfg | 1 + 8 files changed, 369 insertions(+) create mode 100644 cloudkitty/collector/meta.py create mode 100644 cloudkitty/db/sqlalchemy/alembic/versions/2ac2217dcbd9_added_support_for_meta_collector.py diff --git a/cloudkitty/api/controllers/v1.py b/cloudkitty/api/controllers/v1.py index a681da62..206c4c1a 100644 --- a/cloudkitty/api/controllers/v1.py +++ b/cloudkitty/api/controllers/v1.py @@ -24,6 +24,7 @@ import wsmeext.pecan as wsme_pecan from cloudkitty.api.controllers import types as cktypes from cloudkitty import config # noqa +from cloudkitty.db import api as db_api from cloudkitty.openstack.common import log as logging CONF = cfg.CONF @@ -67,6 +68,115 @@ class ResourceDescriptor(wtypes.Base): return sample +class ServiceToCollectorMapping(wtypes.Base): + """Type describing a service to collector mapping. + + """ + + service = wtypes.text + """Name of the service.""" + + collector = wtypes.text + """Name of the collector.""" + + def to_json(self): + res_dict = {} + res_dict[self.service] = self.collector + return res_dict + + @classmethod + def sample(cls): + sample = cls(service='compute', + collector='ceilometer') + return sample + + +class MappingController(rest.RestController): + """REST Controller managing service to collector mapping. + + """ + + def __init__(self): + self._db = db_api.get_instance().get_service_to_collector_mapping() + + @wsme_pecan.wsexpose([wtypes.text]) + def get_all(self): + """Return the list of every services mapped. + + :return: List of every services mapped. + """ + return [mapping.service for mapping in self._db.list_services()] + + @wsme_pecan.wsexpose(ServiceToCollectorMapping, wtypes.text) + def get_one(self, service): + """Return a service to collector mapping. + + :param service: Name of the service to filter on. + """ + try: + return self._db.get_mapping(service) + except db_api.NoSuchMapping as e: + pecan.abort(400, str(e)) + pecan.response.status = 200 + + @wsme_pecan.wsexpose(ServiceToCollectorMapping, + wtypes.text, + body=wtypes.text) + def post(self, service, collector): + """Create or modify a mapping. + + :param service: Name of the service to map a collector to. + :param collector: Name of the collector. + """ + return self._db.set_mapping(service, collector) + + @wsme_pecan.wsexpose(None, body=wtypes.text) + def delete(self, service): + """Delete a mapping. + + :param service: Name of the service to suppress the mapping from. + """ + try: + self._db.delete_mapping(service) + except db_api.NoSuchMapping as e: + pecan.abort(400, str(e)) + pecan.response.status = 204 + + +class CollectorController(rest.RestController): + """REST Controller managing collector modules. + + """ + + mapping = MappingController() + + _custom_actions = { + 'state': ['GET', 'POST'] + } + + def __init__(self): + self._db = db_api.get_instance().get_module_enable_state() + + @wsme_pecan.wsexpose(bool, wtypes.text) + def state(self, collector): + """Query the enable state of a collector. + + :param collector: Name of the collector. + :return: State of the collector. + """ + return self._db.get_state('collector_{}'.format(collector)) + + @wsme_pecan.wsexpose(bool, wtypes.text, body=bool) + def post_state(self, collector, state): + """Set the enable state of a collector. + + :param collector: Name of the collector. + :param state: New state for the collector. + :return: State of the collector. + """ + return self._db.set_state('collector_{}'.format(collector), state) + + class ModulesController(rest.RestController): """REST Controller managing billing modules. @@ -149,5 +259,6 @@ class V1Controller(rest.RestController): """ + collector = CollectorController() billing = BillingController() report = ReportController() diff --git a/cloudkitty/collector/meta.py b/cloudkitty/collector/meta.py new file mode 100644 index 00000000..ea98f017 --- /dev/null +++ b/cloudkitty/collector/meta.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 Objectif Libre +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# @author: Stéphane Albert +# +from stevedore import extension + +from cloudkitty import collector +from cloudkitty.db import api as db_api + +COLLECTORS_NAMESPACE = 'cloudkitty.collector.backends' + + +class MetaCollector(collector.BaseCollector): + def __init__(self, transformers, **kwargs): + super(MetaCollector, self).__init__(transformers, **kwargs) + + self._db = db_api.get_instance().get_service_to_collector_mapping() + + self._collectors = {} + self._load_collectors() + + self._mappings = {} + self._load_mappings() + + def _connect(self): + pass + + def _load_mappings(self): + mappings = self._db.list_services() + for mapping in mappings: + db_mapping = self._db.get_mapping(mapping.service) + self._mappings[db_mapping.service] = db_mapping.collector + + def _check_enabled(self, name): + enable_state = db_api.get_instance().get_module_enable_state() + return enable_state.get_state('collector_{}'.format(name)) + + def _load_collectors(self): + self._collectors = {} + collectors = extension.ExtensionManager( + COLLECTORS_NAMESPACE, + ) + collectors_list = collectors.names() + collectors_list.remove('meta') + + for name in collectors_list: + if self._check_enabled(name): + self._collectors[name] = collectors[name].plugin( + self.transformers, + user=self.user, + password=self.password, + tenant=self.tenant, + region=self.region, + keystone_url=self.keystone_url, + period=self.period) + + def map_retrieve(self, trans_resource, res_collector=None): + if res_collector: + if hasattr(res_collector, trans_resource): + return getattr(res_collector, trans_resource) + for cur_collector in self._collectors.values(): + if hasattr(cur_collector, trans_resource): + return getattr(cur_collector, trans_resource) + + def retrieve(self, resource, start, end=None, project_id=None, + q_filter=None): + + # Resource to function translation + trans_resource = 'get_' + trans_resource += resource.replace('.', '_') + + # Resource to collector mapping processing + res_collector = None + if resource in self._mappings and resource in self._collectors: + res_collector = self._collectors[resource] + + func = self.map_retrieve(trans_resource, res_collector) + if func is not None: + return func(start, end, project_id, q_filter) diff --git a/cloudkitty/db/api.py b/cloudkitty/db/api.py index 1f648ed9..4519e6c8 100644 --- a/cloudkitty/db/api.py +++ b/cloudkitty/db/api.py @@ -89,3 +89,45 @@ class ModuleEnableState(object): :param name: Name of the module :param value: State of the module """ + + +class NoSuchMapping(Exception): + """Raised when the mapping doesn't exist.""" + + def __init__(self, service): + super(NoSuchMapping, self).__init__( + "No such mapping for service: %s" % service) + self.service = service + + +@six.add_metaclass(abc.ABCMeta) +class ServiceToCollectorMapping(object): + """Base class for service to collector mapping.""" + + @abc.abstractmethod + def get_mapping(self, service): + """Get a mapping. + + :return mapping: service to collector object. + """ + + @abc.abstractmethod + def set_mapping(self, service, collector): + """Set a mapping. + + :param service: Service to work on. + :param collector: Collector to prioritize. + :return mapping: Service to Collector object. + """ + + @abc.abstractmethod + def list_services(self): + """Retrieve the list of every services mapped. + + :return list(str): List of services' name. + """ + @abc.abstractmethod + def delete_mapping(self, service): + """Remove a mapping. + + """ diff --git a/cloudkitty/db/sqlalchemy/alembic/versions/2ac2217dcbd9_added_support_for_meta_collector.py b/cloudkitty/db/sqlalchemy/alembic/versions/2ac2217dcbd9_added_support_for_meta_collector.py new file mode 100644 index 00000000..19e3d67e --- /dev/null +++ b/cloudkitty/db/sqlalchemy/alembic/versions/2ac2217dcbd9_added_support_for_meta_collector.py @@ -0,0 +1,30 @@ +"""Added support for meta collector + +Revision ID: 2ac2217dcbd9 +Revises: 464e951dc3b8 +Create Date: 2014-09-25 12:41:28.585333 + +""" + +# revision identifiers, used by Alembic. +revision = '2ac2217dcbd9' +down_revision = '464e951dc3b8' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.create_table('service_to_collector_mappings', + sa.Column('service', sa.String(length=255), nullable=False), + sa.Column('collector', sa.String(length=255), nullable=False), + sa.PrimaryKeyConstraint('service') + ) + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_table('service_to_collector_mappings') + ### end Alembic commands ### diff --git a/cloudkitty/db/sqlalchemy/api.py b/cloudkitty/db/sqlalchemy/api.py index 95fa0870..0b9d6a7b 100644 --- a/cloudkitty/db/sqlalchemy/api.py +++ b/cloudkitty/db/sqlalchemy/api.py @@ -124,6 +124,63 @@ class ModuleEnableState(api.ModuleEnableState): return bool(db_state.state) +class ServiceToCollectorMapping(object): + """Base class for service to collector mapping.""" + + def get_mapping(self, service): + session = db.get_session() + try: + res = utils.model_query( + models.ServiceToCollectorMapping, + session + ).filter_by( + service=service, + ).one() + return res + except sqlalchemy.orm.exc.NoResultFound: + raise api.NoSuchMapping(service) + + def set_mapping(self, service, collector): + session = db.get_session() + with session.begin(): + try: + q = utils.model_query( + models.ServiceToCollectorMapping, + session + ).filter_by( + service=service, + ).with_lockmode('update') + db_mapping = q.one() + db_mapping.collector = collector + except sqlalchemy.orm.exc.NoResultFound: + model = models.ServiceToCollectorMapping + db_mapping = model(service=service, collector=collector) + session.add(db_mapping) + return db_mapping + + def list_services(self): + session = db.get_session() + q = utils.model_query( + models.ServiceToCollectorMapping, + session + ) + res = q.distinct().values( + models.ServiceToCollectorMapping.service + ) + return res + + def delete_mapping(self, service): + session = db.get_session() + r = utils.model_query( + models.ServiceToCollectorMapping, + session + ).filter_by( + service=service, + ).delete() + if not r: + raise api.NoSuchMapping(service) + + class DBAPIManager(object): @staticmethod @@ -134,6 +191,10 @@ class DBAPIManager(object): def get_module_enable_state(): return ModuleEnableState() + @staticmethod + def get_service_to_collector_mapping(): + return ServiceToCollectorMapping() + @staticmethod def get_migration(): return migration diff --git a/cloudkitty/db/sqlalchemy/models.py b/cloudkitty/db/sqlalchemy/models.py index d3cf5034..6d249891 100644 --- a/cloudkitty/db/sqlalchemy/models.py +++ b/cloudkitty/db/sqlalchemy/models.py @@ -66,3 +66,22 @@ class ModuleStateInfo(Base, models.ModelBase): 'enabled={state}>').format( name=self.name, state=self.state) + + +class ServiceToCollectorMapping(Base, models.ModelBase): + """Collector module state. + + """ + + __tablename__ = 'service_to_collector_mappings' + + service = sqlalchemy.Column(sqlalchemy.String(255), + primary_key=True) + collector = sqlalchemy.Column(sqlalchemy.String(255), + nullable=False) + + def __repr__(self): + return ('').format( + service=self.service, + collector=self.collector) diff --git a/doc/source/webapi/v1.rst b/doc/source/webapi/v1.rst index 34bf4fef..7e420117 100644 --- a/doc/source/webapi/v1.rst +++ b/doc/source/webapi/v1.rst @@ -2,6 +2,19 @@ CloudKitty REST API (v1) ======================== +Collector +========= + +.. rest-controller:: cloudkitty.api.controllers.v1:CollectorController + :webprefix: /v1/collector + +.. rest-controller:: cloudkitty.api.controllers.v1:MappingController + :webprefix: /v1/collector/mapping + +.. autotype:: cloudkitty.api.controllers.v1.MetricToCollectorMapping + :members: + + Billing ======= diff --git a/setup.cfg b/setup.cfg index 27352963..d47bfbcd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,6 +26,7 @@ console_scripts = cloudkitty.collector.backends = ceilometer = cloudkitty.collector.ceilometer:CeilometerCollector + meta = cloudkitty.collector.meta:MetaCollector cloudkitty.transformers = CloudKittyFormatTransformer = cloudkitty.transformer.format:CloudKittyFormatTransformer