Refactored meta collector API

Global improvement of the API.
Added support for policy engine.
Added new policy rights to manage the collector mappings.
Updated documentation.
Added contrib script to do simple API tests.

Change-Id: Ibb9dc1e0c7ead75922d777a669e0270b632c4631

cloudkitty/api/v1/controllers/collector.py

@@ -17,6 +17,7 @@
 #
 import pecan
 from pecan import rest
+import six
 from wsme import types as wtypes
 import wsmeext.pecan as wsme_pecan
 
@@ -31,15 +32,6 @@ class MappingController(rest.RestController):
     def __init__(self):
         self._db = db_api.get_instance().get_service_to_collector_mapping()
 
-    @wsme_pecan.wsexpose([wtypes.text])
-    def get_all(self):
-        """Return the list of every services mapped.
-
-        :return: List of every services mapped.
-        """
-        policy.enforce(pecan.request.context, 'collector:list_mappings', {})
-        return [mapping.service for mapping in self._db.list_services()]
-
     @wsme_pecan.wsexpose(collector_models.ServiceToCollectorMapping,
                          wtypes.text)
     def get_one(self, service):
@@ -49,40 +41,105 @@ class MappingController(rest.RestController):
         """
         policy.enforce(pecan.request.context, 'collector:get_mapping', {})
         try:
-            return self._db.get_mapping(service)
+            mapping = self._db.get_mapping(service)
+            return collector_models.ServiceToCollectorMapping(
+                **mapping.as_dict())
         except db_api.NoSuchMapping as e:
-            pecan.abort(400, str(e))
+            pecan.abort(400, six.str_type(e))
+
+    @wsme_pecan.wsexpose(collector_models.ServiceToCollectorMappingCollection,
+                         wtypes.text,
+                         wtypes.text)
+    def get_all(self, collector=None):
+        """Return the list of every services mapped to a collector.
+
+        :param collector: Filter on the collector name.
+        :return: Service to collector mappings collection.
+        """
+        policy.enforce(pecan.request.context, 'collector:list_mappings', {})
+        mappings = [collector_models.ServiceToCollectorMapping(
+            **mapping.as_dict())
+            for mapping in self._db.list_mappings(collector)]
+        return collector_models.ServiceToCollectorMappingCollection(
+            mappings=mappings)
+
+    @wsme_pecan.wsexpose(collector_models.ServiceToCollectorMapping,
+                         wtypes.text,
+                         wtypes.text)
+    def post(self, collector, service):
+        """Create a service to collector mapping.
+
+        :param collector: Name of the collector to apply mapping on.
+        :param service: Name of the service to apply mapping on.
+        """
+        policy.enforce(pecan.request.context, 'collector:manage_mapping', {})
+        new_mapping = self._db.set_mapping(service, collector)
+        return collector_models.ServiceToCollectorMapping(
+            service=new_mapping.service,
+            collector=new_mapping.collector)
+
+    @wsme_pecan.wsexpose(None,
+                         wtypes.text,
+                         wtypes.text,
+                         status_code=204)
+    def delete(self, collector, service):
+        """Delete a service to collector mapping.
+
+        :param collector: Name of the collector to filter on.
+        :param service: Name of the service to filter on.
+        """
+        policy.enforce(pecan.request.context, 'collector:manage_mapping', {})
+        try:
+            self._db.delete_mapping(service)
+        except db_api.NoSuchMapping as e:
+            pecan.abort(400, six.str_type(e))
+
+
+class CollectorStateController(rest.RestController):
+    """REST Controller managing collector states."""
+
+    def __init__(self):
+        self._db = db_api.get_instance().get_module_info()
+
+    @wsme_pecan.wsexpose(collector_models.CollectorInfos, wtypes.text)
+    def get(self, name):
+        """Query the enable state of a collector.
+
+        :param name: Name of the collector.
+        :return: State of the collector.
+        """
+        policy.enforce(pecan.request.context, 'collector:get_state', {})
+        enabled = self._db.get_state('collector_{}'.format(name))
+        collector = collector_models.CollectorInfos(name=name,
+                                                    enabled=enabled)
+        return collector
+
+    @wsme_pecan.wsexpose(collector_models.CollectorInfos,
+                         wtypes.text,
+                         body=collector_models.CollectorInfos)
+    def put(self, name, infos):
+        """Set the enable state of a collector.
+
+        :param name: Name of the collector.
+        :param infos: New state informations of the collector.
+        :return: State of the collector.
+        """
+        policy.enforce(pecan.request.context, 'collector:update_state', {})
+        enabled = self._db.set_state('collector_{}'.format(name),
+                                     infos.enabled)
+        collector = collector_models.CollectorInfos(name=name,
+                                                    enabled=enabled)
+        return collector
 
 
 class CollectorController(rest.RestController):
     """REST Controller managing collector modules."""
 
-    mapping = MappingController()
+    mappings = MappingController()
+    state = CollectorStateController()
 
-    _custom_actions = {
+    # FIXME(sheeprine): Stub function used to pass requests to subcontrollers
-        'state': ['GET', 'POST']
+    @wsme_pecan.wsexpose(None)
-    }
+    def get(self):
-
+        "Unused function, hack to let pecan route requests to subcontrollers."
-    def __init__(self):
+        return
-        self._db = db_api.get_instance().get_module_enable_state()
-
-    @wsme_pecan.wsexpose(bool, wtypes.text)
-    def state(self, collector):
-        """Query the enable state of a collector.
-
-        :param collector: Name of the collector.
-        :return: State of the collector.
-        """
-        policy.enforce(pecan.request.context, 'collector:get_state', {})
-        return self._db.get_state('collector_{}'.format(collector))
-
-    @wsme_pecan.wsexpose(bool, wtypes.text, body=bool)
-    def post_state(self, collector, state):
-        """Set the enable state of a collector.
-
-        :param collector: Name of the collector.
-        :param state: New state for the collector.
-        :return: State of the collector.
-        """
-        policy.enforce(pecan.request.context, 'collector:update_state', {})
-        return self._db.set_state('collector_{}'.format(collector), state)

cloudkitty/api/v1/datamodels/collector.py

@@ -18,6 +18,29 @@
 from wsme import types as wtypes
 
 
+class CollectorInfos(wtypes.Base):
+    """Type describing a collector module.
+
+    """
+
+    name = wtypes.wsattr(wtypes.text, mandatory=False)
+    """Name of the collector."""
+
+    enabled = wtypes.wsattr(bool, mandatory=True)
+    """State of the collector."""
+
+    def to_json(self):
+        res_dict = {'name': self.name,
+                    'enabled': self.enabled}
+        return res_dict
+
+    @classmethod
+    def sample(cls):
+        sample = cls(name='ceilometer',
+                     enabled=True)
+        return sample
+
+
 class ServiceToCollectorMapping(wtypes.Base):
     """Type describing a service to collector mapping.
 
@@ -30,8 +53,8 @@ class ServiceToCollectorMapping(wtypes.Base):
     """Name of the collector."""
 
     def to_json(self):
-        res_dict = {}
+        res_dict = {'service': self.service,
-        res_dict[self.service] = self.collector
+                    'collector': self.collector}
         return res_dict
 
     @classmethod
@@ -39,3 +62,23 @@ class ServiceToCollectorMapping(wtypes.Base):
         sample = cls(service='compute',
                      collector='ceilometer')
         return sample
+
+
+class ServiceToCollectorMappingCollection(wtypes.Base):
+    """Type describing a service to collector mapping collection.
+
+    """
+
+    mappings = [ServiceToCollectorMapping]
+    """List of service to collector mappings."""
+
+    def to_json(self):
+        res_dict = {'mappings': self.mappings}
+        return res_dict
+
+    @classmethod
+    def sample(cls):
+        mapping = ServiceToCollectorMapping(service='compute',
+                                            collector='ceilometer')
+        sample = cls(mappings=[mapping])
+        return sample

cloudkitty/db/api.py

@@ -117,7 +117,7 @@ class NoSuchMapping(Exception):
 
     def __init__(self, service):
         super(NoSuchMapping, self).__init__(
-            "No such mapping for service: %s" % service)
+            "No mapping for service: %s" % service)
         self.service = service
 
 
@@ -142,12 +142,21 @@ class ServiceToCollectorMapping(object):
         """
 
     @abc.abstractmethod
-    def list_services(self):
+    def list_services(self, collector=None):
         """Retrieve the list of every services mapped.
 
+        :param collector: Filter on a collector name.
         :return list(str): List of services' name.
         """
     @abc.abstractmethod
+    def list_mappings(self, collector=None):
+        """Retrieve the list of every mappings.
+
+        :param collector: Filter on a collector's name.
+        :return [tuple(str, str)]: List of mappings.
+        """
+
+    @abc.abstractmethod
     def delete_mapping(self, service):
         """Remove a mapping.
 

cloudkitty/db/sqlalchemy/api.py

@@ -163,7 +163,7 @@ class ServiceToCollectorMapping(object):
             q = utils.model_query(
                 models.ServiceToCollectorMapping,
                 session)
-            q.filter(
+            q = q.filter(
                 models.ServiceToCollectorMapping.service == service)
             return q.one()
         except sqlalchemy.orm.exc.NoResultFound:
@@ -176,8 +176,8 @@ class ServiceToCollectorMapping(object):
                 q = utils.model_query(
                     models.ServiceToCollectorMapping,
                     session)
-                q = q.filter_by(
+                q = q.filter(
-                    service=service)
+                    models.ServiceToCollectorMapping.service == service)
                 q = q.with_lockmode('update')
                 db_mapping = q.one()
                 db_mapping.collector = collector
@@ -188,15 +188,29 @@ class ServiceToCollectorMapping(object):
                 session.add(db_mapping)
         return db_mapping
 
-    def list_services(self):
+    def list_services(self, collector=None):
         session = db.get_session()
         q = utils.model_query(
             models.ServiceToCollectorMapping,
             session)
+        if collector:
+            q = q.filter(
+                models.ServiceToCollectorMapping.collector == collector)
         res = q.distinct().values(
             models.ServiceToCollectorMapping.service)
         return res
 
+    def list_mappings(self, collector=None):
+        session = db.get_session()
+        q = utils.model_query(
+            models.ServiceToCollectorMapping,
+            session)
+        if collector:
+            q = q.filter(
+                models.ServiceToCollectorMapping.collector == collector)
+        res = q.all()
+        return res
+
     def delete_mapping(self, service):
         session = db.get_session()
         q = utils.model_query(

cloudkitty/db/sqlalchemy/models.py

@@ -70,6 +70,12 @@ class ModuleStateInfo(Base, models.ModelBase):
                     name=self.name,
                     state=self.state)
 
+    def as_dict(self):
+        d = {}
+        for c in self.__table__.columns:
+            d[c.name] = self[c.name]
+        return d
+
 
 class ServiceToCollectorMapping(Base, models.ModelBase):
     """Collector module state.
@@ -88,3 +94,9 @@ class ServiceToCollectorMapping(Base, models.ModelBase):
                 'collector={collector}>').format(
                     service=self.service,
                     collector=self.collector)
+
+    def as_dict(self):
+        d = {}
+        for c in self.__table__.columns:
+            d[c.name] = self[c.name]
+        return d

contrib/ci/scripts/test_collector.sh

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+
+show_state()
+{
+    echo ''
+    echo 'Show ceilometer state:'
+    echo "GET http://localhost:8888/v1/collector/ceilometer/state"
+    curl "http://localhost:8888/v1/collector/ceilometer/state"
+    echo ''
+    echo "GET http://localhost:8888/v1/collector/state/ceilometer"
+    curl "http://localhost:8888/v1/collector/state/ceilometer"
+    echo ''
+}
+
+set_state()
+{
+    echo ''
+    echo 'Set ceilometer state:'
+    echo "PUT http://localhost:8888/v1/collector/ceilometer/state"
+    curl "http://localhost:8888/v1/collector/ceilometer/state" \
+    -X PUT -H "Content-Type: application/json" -H "Accept: application/json" \
+    -d '{"enabled": true}'
+    echo ''
+    echo "PUT http://localhost:8888/v1/collector/state/ceilometer"
+    curl "http://localhost:8888/v1/collector/state/ceilometer" \
+    -X PUT -H "Content-Type: application/json" -H "Accept: application/json" \
+    -d '{"enabled": false}'
+    echo ''
+}
+
+list_mappings()
+{
+    echo ''
+    echo 'Get compute mapping:'
+    echo "GET http://localhost:8888/v1/collector/mappings/compute"
+    curl "http://localhost:8888/v1/collector/mappings/compute"
+    echo ''
+
+    echo 'List ceilometer mappings:'
+    echo "GET http://localhost:8888/v1/collector/ceilometer/mappings"
+    curl "http://localhost:8888/v1/collector/ceilometer/mappings"
+    echo ''
+}
+
+set_mappings()
+{
+    echo ''
+    echo 'Set compute to ceilometer mapping:'
+    echo "POST http://localhost:8888/v1/collector/ceilometer/mappings/compute"
+    curl "http://localhost:8888/v1/collector/ceilometer/mappings/compute" \
+    -X POST -H "Content-Type: application/json" -H "Accept: application/json" \
+    -d ''
+    echo ''
+    echo 'Set volume to ceilometer mapping:'
+    echo "POST http://localhost:8888/v1/collector/mappings?collector=ceilometer&service=volume"
+    curl "http://localhost:8888/v1/collector/mappings?collector=ceilometer&service=volume" \
+    -X POST -H "Content-Type: application/json" -H "Accept: application/json" \
+    -d ''
+    echo ''
+}
+
+del_mappings()
+{
+    echo ''
+    echo 'Deleting compute to ceilometer mapping:'
+    echo "DELETE http://localhost:8888/v1/collector/ceilometer/mappings/compute"
+    curl "http://localhost:8888/v1/collector/ceilometer/mappings/compute" \
+    -X DELETE -H "Content-Type: application/json" -H "Accept: application/json" \
+    -d ''
+    test $? && echo 'OK'
+    echo 'Deleting volume to ceilometer mapping:'
+    echo "DELETE http://localhost:8888/v1/collector/mappings?collector=ceilometer&service=volume"
+    curl "http://localhost:8888/v1/collector/mappings?collector=ceilometer&service=volume" \
+    -X DELETE -H "Content-Type: application/json" -H "Accept: application/json" \
+    -d ''
+    test $? && echo 'OK'
+}
+
+show_state
+set_state
+list_mappings
+set_mappings
+list_mappings
+del_mappings
+list_mappings

doc/source/webapi/v1.rst

@@ -11,6 +11,18 @@ Collector
 .. rest-controller:: cloudkitty.api.v1.controllers.collector:MappingController
    :webprefix: /v1/collector/mapping
 
+.. rest-controller:: cloudkitty.api.v1.controllers.collector:CollectorStateController
+   :webprefix: /v1/collector/state
+
+.. autotype:: cloudkitty.api.v1.datamodels.collector.CollectorInfos
+   :members:
+
+.. autotype:: cloudkitty.api.v1.datamodels.collector.ServiceToCollectorMapping
+   :members:
+
+.. autotype:: cloudkitty.api.v1.datamodels.collector.ServiceToCollectorMappingCollection
+   :members:
+
 
 Rating
 ======

etc/cloudkitty/policy.json

@@ -12,6 +12,7 @@
 
     "collector:list_mappings": "role:admin",
     "collector:get_mapping": "role:admin",
+    "collector:manage_mappings": "role:admin",
     "collector:get_state": "role:admin",
     "collector:update_state": "role:admin",