Merge "Improve User Experience by adding an info REST entrypoint"

cloudkitty/api/v1/controllers/__init__.py

@@ -18,6 +18,7 @@
 from pecan import rest
 
 from cloudkitty.api.v1.controllers import collector as collector_api
+from cloudkitty.api.v1.controllers import info as info_api
 from cloudkitty.api.v1.controllers import rating as rating_api
 from cloudkitty.api.v1.controllers import report as report_api
 from cloudkitty.api.v1.controllers import storage as storage_api
@@ -33,3 +34,4 @@ class V1Controller(rest.RestController):
     rating = rating_api.RatingController()
     report = report_api.ReportController()
     storage = storage_api.StorageController()
+    info = info_api.InfoController()

cloudkitty/api/v1/controllers/info.py

@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# @author: Maxime Cottret
+#
+from oslo_config import cfg
+import pecan
+from pecan import rest
+import six
+from wsme import types as wtypes
+import wsmeext.pecan as wsme_pecan
+
+from cloudkitty.api.v1.datamodels import info as info_models
+from cloudkitty.api.v1 import types as ck_types
+from cloudkitty import collector
+from cloudkitty.common import policy
+
+CONF = cfg.CONF
+METADATA = collector.get_collector_metadata()
+
+
+class ServiceInfoController(rest.RestController):
+    """REST Controller mananging collected services information."""
+
+    @wsme_pecan.wsexpose(info_models.CloudkittyServiceInfoCollection)
+    def get_all(self):
+        """Get the service list.
+
+        :return: List of every services.
+        """
+        policy.enforce(pecan.request.context, 'info:list_services_info', {})
+        services_info_list = []
+        for service, metadata in METADATA.items():
+            info = metadata.copy()
+            info['service_id'] = service
+            services_info_list.append(
+                info_models.CloudkittyServiceInfo(**info))
+        return info_models.CloudkittyServiceInfoCollection(
+            services=services_info_list)
+
+    @wsme_pecan.wsexpose(info_models.CloudkittyServiceInfo, wtypes.text)
+    def get_one(self, service_name):
+        """Return a service.
+
+        :param service_name: name of the service.
+        """
+        policy.enforce(pecan.request.context, 'info:get_service_info', {})
+        try:
+            info = METADATA[service_name].copy()
+            info['service_id'] = service_name
+            return info_models.CloudkittyServiceInfo(**info)
+        except KeyError:
+            pecan.abort(404, six.text_type(service_name))
+
+
+class InfoController(rest.RestController):
+    """REST Controller managing Cloudkitty general information."""
+
+    services = ServiceInfoController()
+
+    _custom_actions = {'config': ['GET']}
+
+    @wsme_pecan.wsexpose({
+        str: ck_types.MultiType(wtypes.text, int, float, dict, list)
+    })
+    def config(self):
+        """Return current configuration."""
+        policy.enforce(pecan.request.context, 'info:get_config', {})
+        info = {}
+        info["collect"] = {key: value for key, value in CONF.collect.items()}
+        return info

cloudkitty/api/v1/datamodels/info.py

@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# @author: Stéphane Albert
+#
+from oslo_config import cfg
+from wsme import types as wtypes
+
+CONF = cfg.CONF
+CONF.import_opt('services', 'cloudkitty.collector', 'collect')
+CLOUDKITTY_SERVICES = wtypes.Enum(wtypes.text,
+                                  *CONF.collect.services)
+
+
+class CloudkittyServiceInfo(wtypes.Base):
+    """Type describing a service info in CloudKitty.
+
+    """
+
+    service_id = CLOUDKITTY_SERVICES
+    """Name of the service."""
+
+    metadata = [wtypes.text]
+    """List of service metadata"""
+
+    unit = wtypes.text
+    """service unit"""
+
+    def to_json(self):
+        res_dict = {}
+        res_dict[self.service_id] = [{
+            'metadata': self.metadata,
+            'unit': self.unit
+        }]
+        return res_dict
+
+    @classmethod
+    def sample(cls):
+        sample = cls(service_id='compute',
+                     metadata=['resource_id', 'flavor', 'availability_zone'],
+                     unit='instance')
+        return sample
+
+
+class CloudkittyServiceInfoCollection(wtypes.Base):
+    """A list of CloudKittyServiceInfo."""
+
+    services = [CloudkittyServiceInfo]
+
+    @classmethod
+    def sample(cls):
+        sample = CloudkittyServiceInfo.sample()
+        return cls(services=[sample])

cloudkitty/collector/__init__.py

@@ -66,6 +66,21 @@ def get_collector(transformers=None):
     return collector
 
 
+def get_collector_metadata():
+    """Return dict of metadata.
+
+    Results are based on enabled collector and services in CONF.
+    """
+    transformers = transformer.get_transformers()
+    collector = driver.DriverManager(
+        COLLECTORS_NAMESPACE, CONF.collect.collector,
+        invoke_on_load=False).driver
+    metadata = {}
+    for service in CONF.collect.services:
+        metadata[service] = collector.get_metadata(service, transformers)
+    return metadata
+
+
 class TransformerDependencyError(Exception):
     """Raised when a collector can't find a mandatory transformer."""
 
@@ -132,6 +147,16 @@ class BaseCollector(object):
         trans_resource += resource_name.replace('.', '_')
         return trans_resource
 
+    @classmethod
+    def get_metadata(cls, resource_name, transformers):
+        """Return metadata about collected resource as a dict.
+
+           Dict object should contain:
+                - "metadata": available metadata list,
+                - "unit": collected quantity unit
+        """
+        return {"metadata": [], "unit": "undefined"}
+
     def retrieve(self,
                  resource,
                  start,

cloudkitty/collector/ceilometer.py

@@ -78,6 +78,15 @@ class CeilometerCollector(collector.BaseCollector):
     dependencies = ('CeilometerTransformer',
                     'CloudKittyFormatTransformer')
 
+    units_mappings = {
+        'compute': 'instance',
+        'image': 'MB',
+        'volume': 'GB',
+        'network.bw.out': 'MB',
+        'network.bw.in': 'MB',
+        'network.floating': 'ip',
+    }
+
     def __init__(self, transformers, **kwargs):
         super(CeilometerCollector, self).__init__(transformers, **kwargs)
 
@@ -97,6 +106,18 @@ class CeilometerCollector(collector.BaseCollector):
             '2',
             session=self.session)
 
+    @classmethod
+    def get_metadata(cls, resource_name, transformers):
+        info = super(CeilometerCollector, cls).get_metadata(resource_name,
+                                                            transformers)
+        try:
+            info["metadata"].extend(transformers['CeilometerTransformer']
+                                    .get_metadata(resource_name))
+            info["unit"] = cls.units_mappings[resource_name]
+        except KeyError:
+            pass
+        return info
+
     def gen_filter(self, op='eq', **kwargs):
         """Generate ceilometer filter from kwargs."""
         q_filter = []
@@ -180,9 +201,9 @@ class CeilometerCollector(collector.BaseCollector):
                                                  instance)
             instance = self._cacher.get_resource_detail('compute',
                                                         instance_id)
-            compute_data.append(self.t_cloudkitty.format_item(instance,
+            compute_data.append(
-                                                              'instance',
+                self.t_cloudkitty.format_item(instance, self.units_mappings[
-                                                              1))
+                    "compute"], 1))
         if not compute_data:
             raise collector.NoDataCollected(self.collector_name, 'compute')
         return self.t_cloudkitty.format_service('compute', compute_data)
@@ -205,10 +226,12 @@ class CeilometerCollector(collector.BaseCollector):
                                                  image)
             image = self._cacher.get_resource_detail('image',
                                                      image_id)
+
             image_size_mb = decimal.Decimal(image_stats.max) / units.Mi
-            image_data.append(self.t_cloudkitty.format_item(image,
+            image_data.append(
-                                                            'MB',
+                self.t_cloudkitty.format_item(image, self.units_mappings[
-                                                            image_size_mb))
+                    "image"], image_size_mb))
+
         if not image_data:
             raise collector.NoDataCollected(self.collector_name, 'image')
         return self.t_cloudkitty.format_service('image', image_data)
@@ -232,9 +255,9 @@ class CeilometerCollector(collector.BaseCollector):
                                                  volume)
             volume = self._cacher.get_resource_detail('volume',
                                                       volume_id)
-            volume_data.append(self.t_cloudkitty.format_item(volume,
+            volume_data.append(
-                                                             'GB',
+                self.t_cloudkitty.format_item(volume, self.units_mappings[
-                                                             volume_stats.max))
+                    "volume"], volume_stats.max))
         if not volume_data:
             raise collector.NoDataCollected(self.collector_name, 'volume')
         return self.t_cloudkitty.format_service('volume', volume_data)
@@ -269,10 +292,12 @@ class CeilometerCollector(collector.BaseCollector):
                                                  tap)
             tap = self._cacher.get_resource_detail('network.tap',
                                                    tap_id)
+
             tap_bw_mb = decimal.Decimal(tap_stat.max) / units.M
-            bw_data.append(self.t_cloudkitty.format_item(tap,
+            bw_data.append(
-                                                         'MB',
+                self.t_cloudkitty.format_item(tap, self.units_mappings[
-                                                         tap_bw_mb))
+                    "network.bw." + direction], tap_bw_mb))
+
         ck_res_name = 'network.bw.{}'.format(direction)
         if not bw_data:
             raise collector.NoDataCollected(self.collector_name,
@@ -317,9 +342,9 @@ class CeilometerCollector(collector.BaseCollector):
                                                  floating)
             floating = self._cacher.get_resource_detail('network.floating',
                                                         floating_id)
-            floating_data.append(self.t_cloudkitty.format_item(floating,
+            floating_data.append(
-                                                               'ip',
+                self.t_cloudkitty.format_item(floating, self.units_mappings[
-                                                               1))
+                    "network.floating"], 1))
         if not floating_data:
             raise collector.NoDataCollected(self.collector_name,
                                             'network.floating')

cloudkitty/collector/fake.py

@@ -48,6 +48,24 @@ class CSVCollector(collector.BaseCollector):
         self._file = csvfile
         self._csv = reader
 
+    @classmethod
+    def get_metadata(cls, resource_name, transformers):
+        res = super(CSVCollector, cls).get_metadata(resource_name,
+                                                    transformers)
+        try:
+            filename = cfg.CONF.fake_collector.file
+            csvfile = open(filename, 'rb')
+            reader = csv.DictReader(csvfile)
+            entry = None
+            for row in reader:
+                if row['type'] == resource_name:
+                    entry = row
+                    break
+            res['metadata'] = json.loads(entry['desc']).keys() if entry else {}
+        except IOError:
+            pass
+        return res
+
     def filter_rows(self,
                     start,
                     end=None,

cloudkitty/collector/gnocchi.py

@@ -94,6 +94,18 @@ class GnocchiCollector(collector.BaseCollector):
             '1',
             session=self.session)
 
+    @classmethod
+    def get_metadata(cls, resource_name, transformers):
+        info = super(GnocchiCollector, cls).get_metadata(resource_name,
+                                                         transformers)
+        try:
+            info["metadata"].extend(transformers['GnocchiTransformer']
+                                    .get_metadata(resource_name))
+            info["unit"] = cls.units_mappings[resource_name][1]
+        except KeyError:
+            pass
+        return info
+
     @classmethod
     def gen_filter(cls, cop='=', lop='and', **kwargs):
         """Generate gnocchi filter from kwargs.

cloudkitty/tests/gabbi/gabbits/v1_info.yaml

@@ -0,0 +1,45 @@
+fixtures:
+  - ConfigFixture
+
+tests:
+  - name: get config
+    url: /v1/info/config
+    status: 200
+    response_json_paths:
+      $.collect.services.`len`: 6
+      $.collect.services[0]: compute
+      $.collect.services[1]: image
+      $.collect.services[2]: volume
+      $.collect.services[3]: network.bw.in
+      $.collect.services[4]: network.bw.out
+      $.collect.services[5]: network.floating
+      $.collect.collector: ceilometer
+      $.collect.window: 1800
+      $.collect.wait_periods: 2
+      $.collect.period: 3600
+
+  - name: get services info
+    url: /v1/info/services
+    status: 200
+    response_json_paths:
+      $.services.`len`: 6
+      $.services[/service_id][0].service_id: compute
+      $.services[/service_id][0].unit: instance
+      $.services[/service_id][1].service_id: image
+      $.services[/service_id][1].unit: MB
+      $.services[/service_id][2].service_id: network.bw.in
+      $.services[/service_id][2].unit: MB
+      $.services[/service_id][3].service_id: network.bw.out
+      $.services[/service_id][3].unit: MB
+      $.services[/service_id][4].service_id: network.floating
+      $.services[/service_id][4].unit: ip
+      $.services[/service_id][5].service_id: volume
+      $.services[/service_id][5].unit: GB
+
+  - name: get compute service info
+    url: /v1/info/services/compute
+    status: 200
+    response_json_paths:
+      $.service_id: compute
+      $.unit: instance
+      $.metadata.`len`: 10

cloudkitty/transformer/__init__.py

@@ -64,3 +64,8 @@ class BaseTransformer(object):
         if strip_func:
             return strip_func(res_data)
         return self.generic_strip(res_type, res_data) or res_data
+
+    def get_metadata(self, res_type):
+        """Return list of metadata available for given resource type."""
+
+        return []

cloudkitty/transformer/ceilometer.py

@@ -104,3 +104,26 @@ class CeilometerTransformer(transformer.BaseTransformer):
         res_data['project_id'] = data.project_id
         res_data['floatingip_id'] = data.resource_id
         return res_data
+
+    def get_metadata(self, res_type):
+        """Return list of metadata available after transformation for given
+
+        resource type.
+        """
+
+        class FakeData(dict):
+            """FakeData object."""
+
+            def __getattr__(self, name, default=None):
+                try:
+                    return super(FakeData, self).__getattr__(self, name)
+                except AttributeError:
+                    return default or name
+
+        # list of metadata is built by applying the generic strip_resource_data
+        # function to a fake data object
+
+        fkdt = FakeData()
+        setattr(fkdt, self.metadata_item, FakeData())
+        res_data = self.strip_resource_data(res_type, fkdt)
+        return res_data.keys()

cloudkitty/transformer/gnocchi.py

@@ -55,3 +55,28 @@ class GnocchiTransformer(transformer.BaseTransformer):
             res_data)
         result.update(stripped_data)
         return result
+
+    def get_metadata(self, res_type):
+        """Return list of metadata available after transformation for
+
+        given resource type.
+        """
+
+        class FakeData(dict):
+            """FakeData object."""
+
+            def __getitem__(self, item):
+                try:
+                    return super(FakeData, self).__getitem__(item)
+                except KeyError:
+                    return item
+
+            def get(self, item, default=None):
+                return super(FakeData, self).get(item, item)
+
+        # list of metadata is built by applying the generic strip_resource_data
+        # function to a fake data object
+
+        fkdt = FakeData()
+        res_data = self.strip_resource_data(res_type, fkdt)
+        return res_data.keys()

doc/source/webapi/v1.rst

@@ -24,6 +24,22 @@ Collector
    :members:
 
 
+Info
+====
+
+.. rest-controller:: cloudkitty.api.v1.controllers.info:InfoController
+   :webprefix: /v1/info
+
+.. rest-controller:: cloudkitty.api.v1.controllers.info:ServiceInfoController
+   :webprefix: /v1/info/services
+
+.. autotype:: cloudkitty.api.v1.datamodels.info.CloudkittyServiceInfo
+   :members:
+
+.. autotype:: cloudkitty.api.v1.datamodels.info.CloudkittyServiceInfoCollection
+   :members:
+
+
 Rating
 ======
 

etc/cloudkitty/policy.json

@@ -2,6 +2,10 @@
     "context_is_admin":  "role:admin",
     "default": "",
 
+    "info:list_services_info": "",
+    "info:get_service_info": "",
+    "info:get_config":"",
+
     "rating:list_modules": "role:admin",
     "rating:get_module": "role:admin",
     "rating:update_module": "role:admin",