Allow Cloudkitty to collect non-OpenStack metrics

* Add a SourceFetcher, which allows to retrieve new type of data sources.
  It allows new collectors to be added.
  (examples: Prometheus, Influxdb, Kubernetes...)

* Base Cloudkitty on metrics instead of services and resources
  for collecting metrics. This new architecture allows Cloudkitty
  to be more agnostic and so to rate containers metrics as the same way
  as virtual machines metrics.

* Centralize metrology information in metrics.yml under metrics names.

Task: 6291
Story: 2001501

Change-Id: I00ca080cf05dfc03a3363720f85b79e003eda9be

cloudkitty/api/v1/controllers/info.py

@@ -16,6 +16,7 @@
 # @author: Maxime Cottret
 #
 from oslo_config import cfg
+from oslo_log import log as logging
 import pecan
 from pecan import rest
 import six
@@ -28,51 +29,99 @@ from cloudkitty import collector
 from cloudkitty.common import policy
 from cloudkitty import utils as ck_utils
 
+
+LOG = logging.getLogger(__name__)
+
 CONF = cfg.CONF
 
-METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
 
-METADATA = collector.get_collector_metadata()
+def get_all_metrics():
+    METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
+    METADATA = collector.get_metrics_based_collector_metadata()
+    if 'metrics' not in METRICS_CONF:
+        msg = 'Invalid endpoint: no metrics in current configuration.'
+        pecan.abort(405, msg)
+
+    policy.authorize(pecan.request.context, 'info:list_metrics_info', {})
+    metrics_info_list = []
+    for metric, metadata in METADATA.items():
+        info = metadata.copy()
+        info['metric_id'] = metric
+        metrics_info_list.append(
+            info_models.CloudkittyMetricInfo(**info))
+    return info_models.CloudkittyMetricInfoCollection(
+        metrics=metrics_info_list)
+
+
+def get_one_metric(metric_name):
+    METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
+    METADATA = collector.get_metrics_based_collector_metadata()
+    if 'metrics' not in METRICS_CONF:
+        msg = 'Invalid endpoint: no metrics in current configuration.'
+        pecan.abort(405, msg)
+
+    policy.authorize(pecan.request.context, 'info:get_metric_info', {})
+    try:
+        info = METADATA[metric_name].copy()
+        info['metric_id'] = metric_name
+        return info_models.CloudkittyMetricInfo(**info)
+    except KeyError:
+        pecan.abort(404, six.text_type(metric_name))
+
+
+class MetricInfoController(rest.RestController):
+    """REST Controller managing collected metrics information
+
+    independently of their services.
+    If no metrics are defined in conf, return 405 for each endpoint.
+    """
+
+    @wsme_pecan.wsexpose(info_models.CloudkittyMetricInfoCollection)
+    def get_all(self):
+        """Get the metric list.
+
+        :return: List of every metrics.
+        """
+        return get_all_metrics()
+
+    @wsme_pecan.wsexpose(info_models.CloudkittyMetricInfo, wtypes.text)
+    def get_one(self, metric_name):
+        """Return a metric.
+
+        :param metric_name: name of the metric.
+        """
+        return get_one_metric(metric_name)
 
 
 class ServiceInfoController(rest.RestController):
-    """REST Controller mananging collected services information."""
+    """REST Controller managing collected services information."""
 
-    @wsme_pecan.wsexpose(info_models.CloudkittyServiceInfoCollection)
+    @wsme_pecan.wsexpose(info_models.CloudkittyMetricInfoCollection)
     def get_all(self):
-        """Get the service list.
+        """Get the service list (deprecated).
 
         :return: List of every services.
         """
-        policy.authorize(pecan.request.context, 'info:list_services_info', {})
-        services_info_list = []
-        for service, metadata in METADATA.items():
-            info = metadata.copy()
-            info['service_id'] = service
-            services_info_list.append(
-                info_models.CloudkittyServiceInfo(**info))
-        return info_models.CloudkittyServiceInfoCollection(
-            services=services_info_list)
+        LOG.warning("Services based endpoints are deprecated. "
+                    "Please use metrics based enpoints instead.")
+        return get_all_metrics()
 
-    @wsme_pecan.wsexpose(info_models.CloudkittyServiceInfo, wtypes.text)
+    @wsme_pecan.wsexpose(info_models.CloudkittyMetricInfo, wtypes.text)
     def get_one(self, service_name):
-        """Return a service.
+        """Return a service (deprecated).
 
         :param service_name: name of the service.
         """
-        policy.authorize(pecan.request.context, 'info:get_service_info', {})
-        try:
-            info = METADATA[service_name].copy()
-            info['service_id'] = service_name
-            return info_models.CloudkittyServiceInfo(**info)
-        except KeyError:
-            pecan.abort(404, six.text_type(service_name))
+        LOG.warning("Services based endpoints are deprecated. "
+                    "Please use metrics based enpoints instead.")
+        return get_one_metric(service_name)
 
 
 class InfoController(rest.RestController):
     """REST Controller managing Cloudkitty general information."""
 
     services = ServiceInfoController()
+    metrics = MetricInfoController()
 
     _custom_actions = {'config': ['GET']}
 
@@ -82,6 +131,4 @@ class InfoController(rest.RestController):
     def config(self):
         """Return current configuration."""
         policy.authorize(pecan.request.context, 'info:get_config', {})
-        info = {}
-        info["collect"] = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
-        return info
+        return ck_utils.get_metrics_conf(CONF.collect.metrics_conf)