From 059a94039209653c0ef256a0f076d749381f6822 Mon Sep 17 00:00:00 2001
From: Luka Peschke <luka.peschke@objectif-libre.com>
Date: Tue, 17 Apr 2018 14:49:34 +0200
Subject: [PATCH] Improve metrics configuration

This patch provides a refactoring of the metric
configuration model (and file description) to
improve genericity, maintainability and usage for
non-openstack deployment.

The new metric yaml format is defined in the
attached story task and is validated on load with
voluptuous.

Now, a processor is dedicated to one collector and
one storage backend. Thus, collector and storage
configuration go back to the cloudkitty oslo conf.

Collectors have been refactored to have a code as similar as possible,
in order to ease comprehension for new contributors.

Story: 2001883
Task: 14354
Task: 14355
Task: 14431

Change-Id: I948dd9cd5c113bdaa4e49c532354938ffb45f0e7
---
 cloudkitty/api/v1/controllers/info.py         |  39 ++-
 cloudkitty/api/v1/datamodels/info.py          |  17 +-
 cloudkitty/api/v1/datamodels/rating.py        |  13 +-
 cloudkitty/collector/__init__.py              | 141 ++++++--
 cloudkitty/collector/gnocchi.py               | 250 +++++++++------
 cloudkitty/collector/monasca.py               | 302 +++++++++---------
 cloudkitty/common/config.py                   |   7 +-
 cloudkitty/default_metrics_conf.py            | 124 -------
 cloudkitty/fetcher/__init__.py                |   9 +-
 cloudkitty/fetcher/keystone.py                |  12 +-
 cloudkitty/fetcher/source.py                  |  24 +-
 cloudkitty/orchestrator.py                    |  52 ++-
 cloudkitty/storage/__init__.py                |  17 +-
 cloudkitty/storage/hybrid/backends/gnocchi.py | 144 +++++----
 cloudkitty/tests/gabbi/fixtures.py            |  13 +-
 .../tests/gabbi/gabbits/v1-billing.yaml       |   2 +-
 cloudkitty/tests/gabbi/gabbits/v1-info.yaml   |  60 ++--
 cloudkitty/tests/gabbi/gabbits/v1-rating.yaml |   2 +-
 cloudkitty/tests/gabbi/gabbits/v1-report.yaml |  28 +-
 .../tests/gabbi/gabbits/v1-storage.yaml       |  26 +-
 .../rating/hash/gabbits/hash-errors.yaml      |   8 +-
 .../rating/hash/gabbits/hash-location.yaml    |   4 +-
 .../tests/gabbi/rating/hash/gabbits/hash.yaml |   8 +-
 cloudkitty/tests/samples.py                   | 129 +++++++-
 .../tests/storage/test_hybrid_storage.py      |   7 +-
 cloudkitty/tests/storage/test_storage.py      |  19 +-
 cloudkitty/tests/test_orchestrator.py         |   1 -
 cloudkitty/tests/test_utils.py                |   5 +
 cloudkitty/transformer/format.py              |  14 +-
 cloudkitty/utils.py                           |  81 ++---
 etc/cloudkitty/metrics.yml                    | 137 ++++----
 ...etrics-configuration-271102366f8e6fe7.yaml |   4 +
 requirements.txt                              |   1 +
 setup.cfg                                     |   2 +-
 34 files changed, 894 insertions(+), 808 deletions(-)
 delete mode 100644 cloudkitty/default_metrics_conf.py
 create mode 100644 releasenotes/notes/improve-metrics-configuration-271102366f8e6fe7.yaml

diff --git a/cloudkitty/api/v1/controllers/info.py b/cloudkitty/api/v1/controllers/info.py
index d8b56c35..350e525f 100644
--- a/cloudkitty/api/v1/controllers/info.py
+++ b/cloudkitty/api/v1/controllers/info.py
@@ -20,6 +20,7 @@ from oslo_log import log as logging
 import pecan
 from pecan import rest
 import six
+import voluptuous
 from wsme import types as wtypes
 import wsmeext.pecan as wsme_pecan
 
@@ -36,37 +37,45 @@ CONF = cfg.CONF
 
 
 def get_all_metrics():
-    METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
-    METADATA = collector.get_metrics_based_collector_metadata()
-    if 'metrics' not in METRICS_CONF:
+    try:
+        metrics_conf = collector.validate_conf(
+            ck_utils.load_conf(CONF.collect.metrics_conf))
+    except (voluptuous.Invalid, voluptuous.MultipleInvalid):
         msg = 'Invalid endpoint: no metrics in current configuration.'
         pecan.abort(405, msg)
 
     policy.authorize(pecan.request.context, 'info:list_metrics_info', {})
     metrics_info_list = []
-    for metric, metadata in METADATA.items():
-        info = metadata.copy()
-        info['metric_id'] = metric
+    for metric_name, metric in metrics_conf.items():
+        info = metric.copy()
+        info['metric_id'] = info['alt_name']
         metrics_info_list.append(
             info_models.CloudkittyMetricInfo(**info))
     return info_models.CloudkittyMetricInfoCollection(
         metrics=metrics_info_list)
 
 
+def _find_metric(name, conf):
+    for metric_name, metric in conf.items():
+        if metric['alt_name'] == name:
+            return metric
+
+
 def get_one_metric(metric_name):
-    METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
-    METADATA = collector.get_metrics_based_collector_metadata()
-    if 'metrics' not in METRICS_CONF:
+    try:
+        metrics_conf = collector.validate_conf(
+            ck_utils.load_conf(CONF.collect.metrics_conf))
+    except (voluptuous.Invalid, voluptuous.MultipleInvalid):
         msg = 'Invalid endpoint: no metrics in current configuration.'
         pecan.abort(405, msg)
 
     policy.authorize(pecan.request.context, 'info:get_metric_info', {})
-    try:
-        info = METADATA[metric_name].copy()
-        info['metric_id'] = metric_name
-        return info_models.CloudkittyMetricInfo(**info)
-    except KeyError:
+    metric = _find_metric(metric_name, metrics_conf)
+    if not metric:
         pecan.abort(404, six.text_type(metric_name))
+    info = metric.copy()
+    info['metric_id'] = info['alt_name']
+    return info_models.CloudkittyMetricInfo(**info)
 
 
 class MetricInfoController(rest.RestController):
@@ -131,4 +140,4 @@ class InfoController(rest.RestController):
     def config(self):
         """Return current configuration."""
         policy.authorize(pecan.request.context, 'info:get_config', {})
-        return ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
+        return ck_utils.load_conf(CONF.collect.metrics_conf)
diff --git a/cloudkitty/api/v1/datamodels/info.py b/cloudkitty/api/v1/datamodels/info.py
index c1854fe7..69d94112 100644
--- a/cloudkitty/api/v1/datamodels/info.py
+++ b/cloudkitty/api/v1/datamodels/info.py
@@ -18,29 +18,14 @@
 from oslo_config import cfg
 from wsme import types as wtypes
 
-from cloudkitty.default_metrics_conf import DEFAULT_METRICS_CONF
-from cloudkitty import utils as ck_utils
-
 
 CONF = cfg.CONF
 
 
-def get_metrics_list():
-    metrics_conf = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
-    try:
-        metrics = list(metrics_conf['metrics'].keys())
-        cloudkitty_metrics = wtypes.Enum(wtypes.text, *metrics)
-    except KeyError:
-        metrics = list(DEFAULT_METRICS_CONF['metrics'].keys())
-        cloudkitty_metrics = wtypes.Enum(wtypes.text, *metrics)
-
-    return cloudkitty_metrics
-
-
 class CloudkittyMetricInfo(wtypes.Base):
     """Type describing a metric info in CloudKitty."""
 
-    metric_id = get_metrics_list()
+    metric_id = wtypes.text
     """Name of the metric."""
 
     metadata = [wtypes.text]
diff --git a/cloudkitty/api/v1/datamodels/rating.py b/cloudkitty/api/v1/datamodels/rating.py
index 4de4e818..212e7320 100644
--- a/cloudkitty/api/v1/datamodels/rating.py
+++ b/cloudkitty/api/v1/datamodels/rating.py
@@ -28,16 +28,7 @@ LOG = log.getLogger(__name__)
 
 CONF = cfg.CONF
 
-METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
-
-try:
-    SERVICE_NAMES = list(METRICS_CONF['metrics'].keys())
-except KeyError:
-    LOG.error("No metrics specified in YAML configuration, "
-              "CloudKitty won't work as expected")
-    SERVICE_NAMES = ['compute', 'image']
-
-CLOUDKITTY_SERVICES = wtypes.Enum(wtypes.text, *SERVICE_NAMES)
+METRICS_CONF = ck_utils.load_conf(CONF.collect.metrics_conf)
 
 
 class CloudkittyResource(wtypes.Base):
@@ -45,7 +36,7 @@ class CloudkittyResource(wtypes.Base):
 
     """
 
-    service = CLOUDKITTY_SERVICES
+    service = wtypes.text
     """Name of the service."""
 
     # FIXME(sheeprine): values should be dynamic
diff --git a/cloudkitty/collector/__init__.py b/cloudkitty/collector/__init__.py
index de107a22..897874d8 100644
--- a/cloudkitty/collector/__init__.py
+++ b/cloudkitty/collector/__init__.py
@@ -16,42 +16,105 @@
 # @author: Stéphane Albert
 #
 import abc
+import fractions
 
 from oslo_config import cfg
 import six
 from stevedore import driver
+from voluptuous import All
+from voluptuous import Any
+from voluptuous import Coerce
+from voluptuous import In
+from voluptuous import Invalid
+from voluptuous import Length
+from voluptuous import Optional
+from voluptuous import Required
+from voluptuous import Schema
 
 from cloudkitty import transformer
 from cloudkitty import utils as ck_utils
 
+collect_opts = [
+    cfg.StrOpt('collector',
+               default='gnocchi',
+               help='Data collector.'),
+    cfg.IntOpt('period',
+               default=3600,
+               help='Rating period in seconds.'),
+    cfg.IntOpt('wait_periods',
+               default=2,
+               help='Wait for N periods before collecting new data.'),
+    cfg.StrOpt('metrics_conf',
+               default='/etc/cloudkitty/metrics.yml',
+               help='Metrology configuration file.'),
+]
 
 CONF = cfg.CONF
+CONF.register_opts(collect_opts, 'collect')
 
 COLLECTORS_NAMESPACE = 'cloudkitty.collector.backends'
 
 
+def MetricDict(value):
+    if isinstance(value, dict) and len(value.keys()) > 0:
+        return value
+    raise Invalid("Not a dict with at least one key")
+
+
+CONF_BASE_SCHEMA = {Required('metrics'): MetricDict}
+
+METRIC_BASE_SCHEMA = {
+    # Display unit
+    Required('unit'): All(str, Length(min=1)),
+    # Factor for unit converion
+    Required('factor', default=1):
+        Any(int, float, Coerce(fractions.Fraction)),
+    # Offset for unit conversion
+    Required('offset', default=0):
+        # [int, float, fractions.Fraction],
+        Any(int, float, Coerce(fractions.Fraction)),
+    # Name to be used in dataframes, and used for service creation in hashmap
+    # module. Defaults to the name of the metric
+    Optional('alt_name'): All(str, Length(min=1)),
+    # This is what metrics are grouped by on collection.
+    Required('groupby', default=list): [
+        All(str, Length(min=1))
+    ],
+    # Available in HashMap
+    Required('metadata', default=list): [
+        All(str, Length(min=1))
+    ],
+    # Mutate collected value. May be any of (NONE, NUMBOOL, FLOOR, CEIL).
+    # Defaults to NONE
+    Required('mutate', default='NONE'):
+        In(['NONE', 'NUMBOOL', 'FLOOR', 'CEIL']),
+    # Collector-specific args. Should be overriden by schema provided for
+    # the given collector
+    Optional('extra_args'): dict,
+}
+
+
 def get_collector(transformers=None):
-    metrics_conf = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
+    metrics_conf = ck_utils.load_conf(CONF.collect.metrics_conf)
     if not transformers:
         transformers = transformer.get_transformers()
     collector_args = {
-        'period': metrics_conf.get('period', 3600),
+        'period': CONF.collect.period,
         'transformers': transformers,
     }
     collector_args.update({'conf': metrics_conf})
     return driver.DriverManager(
         COLLECTORS_NAMESPACE,
-        metrics_conf.get('collector', 'gnocchi'),
+        CONF.collect.collector,
         invoke_on_load=True,
         invoke_kwds=collector_args).driver
 
 
 def get_collector_without_invoke():
     """Return the collector without invoke it."""
-    metrics_conf = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
     return driver.DriverManager(
         COLLECTORS_NAMESPACE,
-        metrics_conf.get('collector', 'gnocchi'),
+        CONF.collect.collector,
         invoke_on_load=False
     ).driver
 
@@ -61,14 +124,15 @@ def get_metrics_based_collector_metadata():
 
     Results are based on enabled collector and metrics in CONF.
     """
-    metrics_conf = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
+    metrics_conf = ck_utils.load_conf(CONF.collect.metrics_conf)
     transformers = transformer.get_transformers()
     collector = get_collector_without_invoke()
     metadata = {}
     if 'metrics' in metrics_conf:
-        for metric in metrics_conf.get('metrics', {}):
-            metadata[metric] = collector.get_metadata(
-                metric,
+        for metric_name, metric in metrics_conf.get('metrics', {}).items():
+            alt_name = metric.get('alt_name', metric_name)
+            metadata[alt_name] = collector.get_metadata(
+                metric_name,
                 transformers,
                 metrics_conf,
             )
@@ -102,17 +166,18 @@ class NoDataCollected(Exception):
 @six.add_metaclass(abc.ABCMeta)
 class BaseCollector(object):
     collector_name = None
-    dependencies = []
+    dependencies = ['CloudKittyFormatTransformer']
 
     def __init__(self, transformers, **kwargs):
         try:
             self.transformers = transformers
             self.period = kwargs['period']
-            self.conf = kwargs['conf']
-        except IndexError as e:
+            self.conf = self.check_configuration(kwargs['conf'])
+        except KeyError as e:
             raise ValueError("Missing argument (%s)" % e)
 
         self._check_transformers()
+        self.t_cloudkitty = self.transformers['CloudKittyFormatTransformer']
 
     def _check_transformers(self):
         """Check for transformer prerequisites
@@ -123,6 +188,13 @@ class BaseCollector(object):
                 raise TransformerDependencyError(self.collector_name,
                                                  dependency)
 
+    @staticmethod
+    def check_configuration(self, conf):
+        """Check metrics configuration
+
+        """
+        return Schema(METRIC_BASE_SCHEMA)(conf)
+
     @staticmethod
     def last_month():
         month_start = ck_utils.get_month_start()
@@ -152,16 +224,35 @@ class BaseCollector(object):
         """
         return {"metadata": [], "unit": "undefined"}
 
-    def retrieve(self,
-                 resource,
-                 start,
-                 end=None,
-                 project_id=None,
-                 q_filter=None):
-        trans_resource = self._res_to_func(resource)
-        if not hasattr(self, trans_resource):
-            raise NotImplementedError(
-                "No method found in collector '%s' for resource '%s'."
-                % (self.collector_name, resource))
-        func = getattr(self, trans_resource)
-        return func(resource, start, end, project_id, q_filter)
+    @abc.abstractmethod
+    def fetch_all(self, metric_name, start, end,
+                  project_id=None, q_filter=None):
+        pass
+
+    def retrieve(self, metric_name, start, end,
+                 project_id=None, q_filter=None):
+
+        data = self.fetch_all(
+            metric_name,
+            start,
+            end,
+            project_id,
+            q_filter=q_filter,
+        )
+
+        name = self.conf[metric_name].get('alt_name', metric_name)
+        if data:
+            data = self.t_cloudkitty.format_service(name, data)
+        if not data:
+            raise NoDataCollected(self.collector_name, name)
+        return data
+
+
+def validate_conf(conf):
+    """Validates the provided configuration."""
+    collector = get_collector_without_invoke()
+    output = collector.check_configuration(conf)
+    for metric_name, metric in output.items():
+        if 'alt_name' not in metric.keys():
+            metric['alt_name'] = metric_name
+    return output
diff --git a/cloudkitty/collector/gnocchi.py b/cloudkitty/collector/gnocchi.py
index 822db91f..ae760a31 100644
--- a/cloudkitty/collector/gnocchi.py
+++ b/cloudkitty/collector/gnocchi.py
@@ -13,13 +13,16 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 #
-import decimal
-
 from gnocchiclient import auth as gauth
 from gnocchiclient import client as gclient
 from keystoneauth1 import loading as ks_loading
 from oslo_config import cfg
 from oslo_log import log as logging
+from voluptuous import All
+from voluptuous import In
+from voluptuous import Length
+from voluptuous import Required
+from voluptuous import Schema
 
 from cloudkitty import collector
 from cloudkitty import utils as ck_utils
@@ -65,10 +68,25 @@ ks_loading.register_auth_conf_options(
 CONF = cfg.CONF
 
 
+GNOCCHI_EXTRA_SCHEMA = {
+    Required('extra_args'): {
+        Required('resource_type'): All(str, Length(min=1)),
+        # Due to Gnocchi model, metric are grouped by resource.
+        # This parameter permits to adapt the key of the resource identifier
+        Required('resource_key', default='id'): All(str, Length(min=1)),
+        # This is needed to allow filtering on the project for the Openstack
+        # usecase.
+        # NOTE(MCO): maybe be removed in following releases
+        Required('scope_key', default='project_id'): All(str, Length(min=1)),
+        Required('aggregation_method', default='max'):
+            In(['max', 'mean', 'min']),
+    },
+}
+
+
 class GnocchiCollector(collector.BaseCollector):
+
     collector_name = 'gnocchi'
-    dependencies = ('GnocchiTransformer',
-                    'CloudKittyFormatTransformer')
 
     def __init__(self, transformers, **kwargs):
         super(GnocchiCollector, self).__init__(transformers, **kwargs)
@@ -94,14 +112,34 @@ class GnocchiCollector(collector.BaseCollector):
             adapter_options=adapter_options,
         )
 
+    @staticmethod
+    def check_configuration(conf):
+        """Check metrics configuration
+
+        """
+        conf = Schema(collector.CONF_BASE_SCHEMA)(conf)
+        metric_schema = Schema(collector.METRIC_BASE_SCHEMA).extend(
+            GNOCCHI_EXTRA_SCHEMA)
+
+        output = dict()
+        for metric_name, metric in conf['metrics'].items():
+            output[metric_name] = metric_schema(metric)
+            output[metric_name]['groupby'].append(
+                output[metric_name]['extra_args']['resource_key']
+            )
+        return output
+
     @classmethod
     def get_metadata(cls, resource_name, transformers, conf):
         info = super(GnocchiCollector, cls).get_metadata(resource_name,
                                                          transformers)
         try:
-            info["metadata"].extend(transformers['GnocchiTransformer']
-                                    .get_metadata(resource_name))
-            info['unit'] = conf['metrics'][resource_name]['unit']
+            info["metadata"].extend(
+                conf[resource_name]['groupby']
+            ).extend(
+                conf[resource_name]['metadata']
+            )
+            info['unit'] = conf[resource_name]['unit']
         except KeyError:
             pass
         return info
@@ -154,38 +192,43 @@ class GnocchiCollector(collector.BaseCollector):
             self.gen_filter(cop="<=", started_at=end))
         return time_filter
 
-    def _expand(self, metrics, resource, name, aggregate, start, end):
-        try:
-            values = self._conn.metric.get_measures(
-                metric=metrics[name],
-                start=ck_utils.ts2dt(start),
-                stop=ck_utils.ts2dt(end),
-                aggregation=aggregate)
-            # NOTE(sheeprine): Get the list of values for the current
-            # metric and get the first result value.
-            # [point_date, granularity, value]
-            # ["2015-11-24T00:00:00+00:00", 86400.0, 64.0]
-            resource[name] = values[0][2]
-        except (IndexError, KeyError):
-            resource[name] = 0
-
-    def _expand_metrics(self, resources, mappings, start, end, resource_name):
-        for resource in resources:
-            metrics = resource.get('metrics', {})
-            self._expand(
-                metrics,
-                resource,
-                resource_name,
-                mappings,
-                start,
-                end,
-            )
-
-    def get_resources(self, resource_name, start, end,
-                      project_id, q_filter=None):
+    def _fetch_resources(self, metric_name, start, end,
+                         project_id=None, q_filter=None):
         """Get resources during the timeframe.
 
-        :param resource_name: Resource name to filter on.
+        :type metric_name: str
+        :param start: Start of the timeframe.
+        :param end: End of the timeframe if needed.
+        :param project_id: Filter on a specific tenant/project.
+        :type project_id: str
+        :param q_filter: Append a custom filter.
+        :type q_filter: list
+        """
+
+        # Get gnocchi specific conf
+        extra_args = self.conf[metric_name]['extra_args']
+        # Build query
+        query_parameters = self._generate_time_filter(start, end)
+
+        resource_type = extra_args['resource_type']
+
+        query_parameters.append(
+            self.gen_filter(cop="=", type=resource_type))
+        if project_id:
+            kwargs = {extra_args['scope_key']: project_id}
+            query_parameters.append(self.gen_filter(**kwargs))
+        if q_filter:
+            query_parameters.append(q_filter)
+        resources = self._conn.resource.search(
+            resource_type=resource_type,
+            query=self.extend_filter(*query_parameters))
+        return {res[extra_args['resource_key']]: res for res in resources}
+
+    def _fetch_metric(self, metric_name, start, end,
+                      project_id=None, q_filter=None):
+        """Get metric during the timeframe.
+
+        :param metric_name: metric name to filter on.
         :type resource_name: str
         :param start: Start of the timeframe.
         :param end: End of the timeframe if needed.
@@ -194,89 +237,94 @@ class GnocchiCollector(collector.BaseCollector):
         :param q_filter: Append a custom filter.
         :type q_filter: list
         """
-        # NOTE(sheeprine): We first get the list of every resource running
-        # without any details or history.
-        # Then we get information about the resource getting details and
-        # history.
 
-        # Translating the resource name if needed
-        query_parameters = self._generate_time_filter(start, end)
+        # Get gnocchi specific conf
+        extra_args = self.conf[metric_name]['extra_args']
 
-        resource_type = self.conf['metrics'][resource_name]['resource']
+        # get ressource type
+        resource_type = extra_args['resource_type']
 
+        # build search query using ressource type and project_id if provided
+        query_parameters = list()
         query_parameters.append(
             self.gen_filter(cop="=", type=resource_type))
-        query_parameters.append(
-            self.gen_filter(project_id=project_id))
+        if project_id:
+            kwargs = {extra_args['scope_key']: project_id}
+            query_parameters.append(self.gen_filter(**kwargs))
         if q_filter:
             query_parameters.append(q_filter)
-        resources = self._conn.resource.search(
+
+        # build aggregration operation
+        op = ["aggregate", extra_args['aggregation_method'],
+              ["metric", metric_name, extra_args['aggregation_method']]]
+
+        # get groupby
+        groupby = self.conf[metric_name]['groupby']
+
+        return self._conn.aggregates.fetch(
+            op,
             resource_type=resource_type,
-            query=self.extend_filter(*query_parameters))
-        return resources
+            start=ck_utils.ts2dt(start),
+            stop=ck_utils.ts2dt(end),
+            groupby=groupby,
+            search=self.extend_filter(*query_parameters))
 
-    def resource_info(self, resource_name, start, end,
-                      project_id, q_filter=None):
-        met = self.conf['metrics'][resource_name]
-        unit = met['unit']
-        qty = 1 if met.get('countable_unit') else met['resource']
+    def _format_data(self, metconf, data, resources_info=None):
+        """Formats gnocchi data to CK data.
 
-        resources = self.get_resources(
-            resource_name,
+        Returns metadata, groupby and qty
+
+        """
+        groupby = data['group']
+        # if resource info is provided, add additional
+        # metadata as defined in the conf
+        metadata = dict()
+        if resources_info:
+            resource = resources_info[
+                groupby[metconf['extra_args']['resource_key']]]
+            for i in metconf['metadata']:
+                metadata[i] = resource.get(i, '')
+
+        qty = data['measures']['measures']['aggregated'][0][2]
+        converted_qty = ck_utils.convert_unit(
+            qty, metconf['factor'], metconf['offset'])
+        mutated_qty = ck_utils.mutate(converted_qty, metconf['mutate'])
+        return metadata, groupby, mutated_qty
+
+    def fetch_all(self, metric_name, start, end,
+                  project_id=None, q_filter=None):
+
+        met = self.conf[metric_name]
+
+        data = self._fetch_metric(
+            metric_name,
             start,
             end,
             project_id=project_id,
             q_filter=q_filter,
         )
 
-        formated_resources = list()
-        for resource in resources:
-            resource_data = self.t_gnocchi.strip_resource_data(
-                resource_name, resource)
-
-            mapp = self.conf['metrics'][resource_name]['aggregation_method']
-
-            self._expand_metrics(
-                [resource_data],
-                mapp,
+        resources_info = None
+        if met['metadata']:
+            resources_info = self._fetch_resources(
+                metric_name,
                 start,
                 end,
-                resource_name,
+                project_id=project_id,
+                q_filter=q_filter
             )
 
-            resource_data.pop('metrics', None)
-
-            # Unit conversion
-            if isinstance(qty, str):
-                resource_data[resource_name] = ck_utils.convert_unit(
-                    resource_data[resource_name],
-                    self.conf['metrics'][resource_name].get('factor', 1),
-                    self.conf['metrics'][resource_name].get('offset', 0),
+        formated_resources = list()
+        for d in data:
+            # Only if aggregates have been found
+            if d['measures']['measures']['aggregated']:
+                metadata, groupby, qty = self._format_data(
+                    met, d, resources_info)
+                data = self.t_cloudkitty.format_item(
+                    groupby,
+                    metadata,
+                    met['unit'],
+                    qty=qty,
                 )
-
-            val = qty if isinstance(qty, int) else resource_data[resource_name]
-            data = self.t_cloudkitty.format_item(
-                resource_data,
-                unit,
-                decimal.Decimal(val)
-            )
-
-            # NOTE(sheeprine): Reference to gnocchi resource used by storage
-            data['resource_id'] = data['desc']['resource_id']
-            formated_resources.append(data)
+                formated_resources.append(data)
         return formated_resources
-
-    def retrieve(self, resource_name, start, end,
-                 project_id, q_filter=None):
-
-        resources = self.resource_info(
-            resource_name,
-            start,
-            end,
-            project_id,
-            q_filter=q_filter,
-        )
-
-        if not resources:
-            raise collector.NoDataCollected(self.collector_name, resource_name)
-        return self.t_cloudkitty.format_service(resource_name, resources)
diff --git a/cloudkitty/collector/monasca.py b/cloudkitty/collector/monasca.py
index 96706463..9f6ada39 100644
--- a/cloudkitty/collector/monasca.py
+++ b/cloudkitty/collector/monasca.py
@@ -15,13 +15,16 @@
 #
 # @author: Luka Peschke
 #
-import decimal
-
 from keystoneauth1 import loading as ks_loading
 from keystoneclient.v3 import client as ks_client
 from monascaclient import client as mclient
 from oslo_config import cfg
 from oslo_log import log as logging
+from voluptuous import All
+from voluptuous import In
+from voluptuous import Length
+from voluptuous import Required
+from voluptuous import Schema
 
 from cloudkitty import collector
 from cloudkitty import transformer
@@ -43,7 +46,22 @@ ks_loading.register_auth_conf_options(
     COLLECTOR_MONASCA_OPTS)
 CONF = cfg.CONF
 
-METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
+METRICS_CONF = ck_utils.load_conf(CONF.collect.metrics_conf)
+
+MONASCA_EXTRA_SCHEMA = {
+    Required('extra_args'): {
+        # Key corresponding to the resource id in a metric's dimensions
+        # Allows to adapt the resource identifier. Should not need to be
+        # modified in a standard OpenStack installation
+        Required('resource_key', default='resource_id'):
+            All(str, Length(min=1)),
+        # This is needed to allow filtering on the project for the Openstack
+        # usecase. May be removed in following releases
+        Required('scope_key', default='project_id'): All(str, Length(min=1)),
+        Required('aggregation_method', default='max'):
+            In(['max', 'mean', 'min']),
+    },
+}
 
 
 class EndpointNotFound(Exception):
@@ -53,13 +71,24 @@ class EndpointNotFound(Exception):
 
 class MonascaCollector(collector.BaseCollector):
     collector_name = 'monasca'
-    dependencies = ['CloudKittyFormatTransformer']
+
+    @staticmethod
+    def check_configuration(conf):
+        """Check metrics configuration
+
+        """
+        conf = Schema(collector.CONF_BASE_SCHEMA)(conf)
+        metric_schema = Schema(collector.METRIC_BASE_SCHEMA).extend(
+            MONASCA_EXTRA_SCHEMA)
+
+        output = dict()
+        for metric_name, metric in conf['metrics'].items():
+            output[metric_name] = metric_schema(metric)
+        return output
 
     def __init__(self, transformers, **kwargs):
         super(MonascaCollector, self).__init__(transformers, **kwargs)
 
-        self.t_cloudkitty = self.transformers['CloudKittyFormatTransformer']
-
         self.auth = ks_loading.load_auth_from_conf_options(
             CONF,
             COLLECTOR_MONASCA_OPTS)
@@ -90,28 +119,13 @@ class MonascaCollector(collector.BaseCollector):
                 return endpoint.url
         return None
 
-    def _get_metadata(self, resource_type, transformers, conf):
+    def _get_metadata(self, metric_name, transformers, conf):
         info = {}
-        info['unit'] = conf['metrics'][resource_type]['unit']
+        info['unit'] = conf['metrics'][metric_name]['unit']
 
-        start = ck_utils.dt2ts(ck_utils.get_month_start())
-        end = ck_utils.dt2ts(ck_utils.get_month_end())
-
-        try:
-            resource = self.active_resources(
-                resource_type,
-                start,
-                end,
-                None,
-            )[0]
-        except IndexError:
-            resource = {}
-        info['metadata'] = resource.get('dimensions', {}).keys()
-
-        service_metrics = METRICS_CONF['services_metrics'][resource_type]
-        for service_metric in service_metrics:
-            metric, statistics = list(service_metric.items())[0]
-            info['metadata'].append(metric)
+        dimension_names = self._conn.metric.list_dimension_names(
+            metric_name=metric_name)
+        info['metadata'] = [d['dimension_name'] for d in dimension_names]
         return info
 
     # NOTE(lukapeschke) if anyone sees a better way to do this,
@@ -124,144 +138,124 @@ class MonascaCollector(collector.BaseCollector):
         tmp = cls(**args)
         return tmp._get_metadata(resource_type, transformers, conf)
 
-    def _get_resource_metadata(self, resource_type, start,
-                               end, resource_id, conf):
-        meter = conf['metrics'][resource_type]['resource']
-
-        if not meter:
-            return {}
-        measurements = self._conn.metrics.list_measurements(
-            name=meter,
-            start_time=ck_utils.ts2dt(start),
-            end_time=ck_utils.ts2dt(end),
-            merge_metrics=True,
-            dimensions={'resource_id': resource_id},
-        )
-        try:
-            # Getting the last measurement of given period
-            metadata = measurements[-1]['measurements'][-1][2]
-        except (KeyError, IndexError):
-            metadata = {}
-        return metadata
-
-    def _get_resource_qty(self, meter, start, end, resource_id, statistics):
-        # NOTE(lukapeschke) the period trick is used to aggregate
-        # the measurements
-        period = end - start
-        statistics = self._conn.metrics.list_statistics(
-            name=meter,
-            start_time=ck_utils.ts2dt(start),
-            end_time=ck_utils.ts2dt(end),
-            dimensions={'resource_id': resource_id},
-            statistics=statistics,
-            period=period,
-            merge_metrics=True,
-        )
-        try:
-            # If several statistics are returned (should not happen),
-            # use the latest
-            qty = decimal.Decimal(statistics[-1]['statistics'][-1][1])
-        except (KeyError, IndexError):
-            qty = decimal.Decimal(0)
-        return qty
-
-    def _is_resource_active(self, meter, resource_id, start, end):
-        measurements = self._conn.metrics.list_measurements(
-            name=meter,
-            start_time=ck_utils.ts2dt(start),
-            end_time=ck_utils.ts2dt(end),
-            group_by='resource_id',
-            merge_metrics=True,
-            dimensions={'resource_id': resource_id},
-        )
-        return len(measurements) > 0
-
-    def active_resources(self, resource_type, start,
-                         end, project_id, conf, **kwargs):
-        meter = conf['metrics'][resource_type]['resource']
-
-        if not meter:
-            return {}
+    def _get_dimensions(self, metric_name, project_id, q_filter):
+        extra_args = self.conf[metric_name]['extra_args']
         dimensions = {}
         if project_id:
-            dimensions['project_id'] = project_id
-        dimensions.update(kwargs)
-        resources = self._conn.metrics.list(name=meter, dimensions=dimensions)
-        output = []
-        for resource in resources:
-            try:
-                resource_id = resource['dimensions']['resource_id']
-                if (resource_id not in
-                    [item['dimensions']['resource_id'] for item in output]
-                        and self._is_resource_active(meter, resource_id,
-                                                     start, end)):
-                    output.append(resource)
-            except KeyError:
-                continue
-        return output
+            dimensions[extra_args['scope_key']] = project_id
+        if q_filter:
+            dimensions.update(q_filter)
+        return dimensions
 
-    def _expand_metrics(self, resource, resource_id,
-                        mappings, start, end, resource_type):
-        for mapping in mappings:
-            name, statistics = list(mapping.items())[0]
-            qty = self._get_resource_qty(
-                name,
-                start,
-                end,
-                resource_id,
-                statistics,
-            )
+    def _fetch_measures(self, metric_name, start, end,
+                        project_id=None, q_filter=None):
+        """Get measures for given metric during the timeframe.
 
-            conv_data = METRICS_CONF['metrics'][resource_type].get(name)
-            if conv_data:
-                resource[name] = ck_utils.convert_unit(
-                    qty,
-                    conv_data.get('factor', 1),
-                    conv_data.get('offset', 0),
-                )
+        :param metric_name: metric name to filter on.
+        :type metric_name: str
+        :param start: Start of the timeframe.
+        :param end: End of the timeframe if needed.
+        :param project_id: Filter on a specific tenant/project.
+        :type project_id: str
+        :param q_filter: Append a custom filter.
+        :type q_filter: list
+        """
 
-    def resource_info(self, resource_type, start, end,
-                      project_id, q_filter=None):
-        met = self.conf['metrics'][resource_type]
-        unit = met['unit']
-        qty = 1 if met.get('countable_unit') else met['resource']
+        dimensions = self._get_dimensions(metric_name, project_id, q_filter)
+        group_by = self.conf[metric_name]['groupby']
+        # NOTE(lpeschke): One aggregated measure per collect period
+        period = end - start
 
-        active_resources = self.active_resources(
-            resource_type, start, end, project_id
+        extra_args = self.conf[metric_name]['extra_args']
+        return self._conn.metrics.list_statistics(
+            name=metric_name,
+            merge_metrics=True,
+            dimensions=dimensions,
+            start_time=ck_utils.ts2dt(start),
+            end_time=ck_utils.ts2dt(end),
+            period=period,
+            statistics=extra_args['aggregation_method'],
+            group_by=group_by)
+
+    def _fetch_metrics(self, metric_name, start, end,
+                       project_id=None, q_filter=None):
+        """List active metrics during the timeframe.
+
+        :param metric_name: metric name to filter on.
+        :type metric_name: str
+        :param start: Start of the timeframe.
+        :param end: End of the timeframe if needed.
+        :param project_id: Filter on a specific tenant/project.
+        :type project_id: str
+        :param q_filter: Append a custom filter.
+        :type q_filter: list
+        """
+        dimensions = self._get_dimensions(metric_name, project_id, q_filter)
+        metrics = self._conn.metrics.list(
+            name=metric_name,
+            dimensions=dimensions,
+            start_time=ck_utils.ts2dt(start),
+            end_time=ck_utils.ts2dt(end),
         )
 
-        resource_data = []
-        for resource in active_resources:
-            resource_id = resource['dimensions']['resource_id']
-            data = resource['dimensions']
-            mappings = (
-                resource_type,
-                METRICS_CONF['metrics'][resource_type]['aggregation_method'],
-            )
+        resource_key = self.conf[metric_name]['extra_args']['resource_key']
 
-            self._expand_metrics(
-                data,
-                resource_id,
-                mappings,
+        return {metric['dimensions'][resource_key]:
+                metric['dimensions'] for metric in metrics}
+
+    def _format_data(self, metconf, data, resources_info=None):
+        """Formats Monasca data to CK data.
+
+        Returns metadata, groupby and qty
+
+        """
+        groupby = data['dimensions']
+
+        resource_key = metconf['extra_args']['resource_key']
+        metadata = dict()
+        if resources_info:
+            resource = resources_info[groupby[resource_key]]
+            for i in metconf['metadata']:
+                metadata[i] = resource.get(i, '')
+
+        qty = data['statistics'][0][1]
+        converted_qty = ck_utils.convert_unit(
+            qty, metconf['factor'], metconf['offset'])
+        mutated_qty = ck_utils.mutate(converted_qty, metconf['mutate'])
+        return metadata, groupby, mutated_qty
+
+    def fetch_all(self, metric_name, start, end,
+                  project_id=None, q_filter=None):
+        met = self.conf[metric_name]
+
+        data = self._fetch_measures(
+            metric_name,
+            start,
+            end,
+            project_id=project_id,
+            q_filter=q_filter,
+        )
+
+        resources_info = None
+        if met['metadata']:
+            resources_info = self._fetch_metrics(
+                metric_name,
                 start,
                 end,
-                resource_type,
+                project_id=project_id,
+                q_filter=q_filter,
             )
-            resource_qty = qty
-            if not (isinstance(qty, int) or isinstance(qty, decimal.Decimal)):
-                resource_qty = METRICS_CONF['services_objects'][resource_type]
-                resource_qty = data[resource_qty]
 
-            resource = self.t_cloudkitty.format_item(data, unit, resource_qty)
-            resource['desc']['resource_id'] = resource_id
-            resource['resource_id'] = resource_id
-            resource_data.append(resource)
-        return resource_data
-
-    def retrieve(self, resource_type, start, end, project_id, q_filter=None):
-        resources = self.resource_info(resource_type, start, end,
-                                       project_id=project_id,
-                                       q_filter=q_filter)
-        if not resources:
-            raise collector.NoDataCollected(self.collector_name, resource_type)
-        return self.t_cloudkitty.format_service(resource_type, resources)
+        formated_resources = list()
+        for d in data:
+            if len(d['statistics']):
+                metadata, groupby, qty = self._format_data(
+                    met, d, resources_info)
+                data = self.t_cloudkitty.format_item(
+                    groupby,
+                    metadata,
+                    met['unit'],
+                    qty=qty,
+                )
+                formated_resources.append(data)
+        return formated_resources
diff --git a/cloudkitty/common/config.py b/cloudkitty/common/config.py
index b2237ab8..916a7a05 100644
--- a/cloudkitty/common/config.py
+++ b/cloudkitty/common/config.py
@@ -22,6 +22,7 @@ import cloudkitty.collector.monasca
 import cloudkitty.config
 import cloudkitty.fetcher
 import cloudkitty.fetcher.keystone
+import cloudkitty.fetcher.source
 import cloudkitty.orchestrator
 import cloudkitty.service
 import cloudkitty.storage
@@ -34,7 +35,7 @@ _opts = [
     ('api', list(itertools.chain(
         cloudkitty.api.app.api_opts,))),
     ('collect', list(itertools.chain(
-        cloudkitty.utils.collect_opts))),
+        cloudkitty.collector.collect_opts))),
     ('collector_monasca', list(itertools.chain(
         cloudkitty.collector.monasca.collector_monasca_opts))),
     ('gnocchi_collector', list(itertools.chain(
@@ -43,6 +44,8 @@ _opts = [
     ('keystone_fetcher', list(itertools.chain(
         cloudkitty.fetcher.keystone.keystone_fetcher_opts,
         cloudkitty.fetcher.keystone.keystone_common_opts))),
+    ('source_fetcher', list(itertools.chain(
+        cloudkitty.fetcher.source.source_fetcher_opts))),
     ('orchestrator', list(itertools.chain(
         cloudkitty.orchestrator.orchestrator_opts))),
     ('output', list(itertools.chain(
@@ -50,7 +53,7 @@ _opts = [
     ('state', list(itertools.chain(
         cloudkitty.config.state_opts))),
     ('storage', list(itertools.chain(
-        cloudkitty.utils.storage_opts))),
+        cloudkitty.storage.storage_opts))),
     ('storage_gnocchi', list(itertools.chain(
         cloudkitty.storage.hybrid.backends.gnocchi.gnocchi_storage_opts))),
     ('fetcher', list(itertools.chain(
diff --git a/cloudkitty/default_metrics_conf.py b/cloudkitty/default_metrics_conf.py
deleted file mode 100644
index 209b4e3b..00000000
--- a/cloudkitty/default_metrics_conf.py
+++ /dev/null
@@ -1,124 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2018 Objectif Libre
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-#
-# @author: Martin CAMEY
-#
-DEFAULT_METRICS_CONF = {
-    'name': 'OpenStack',
-
-    'fetcher': 'keystone',
-    'collector': 'gnocchi',
-
-    'period': 3600,
-    'wait_periods': 2,
-    'window': 1800,
-
-    'services_objects': {
-        'compute': 'instance',
-        'volume': 'volume',
-        'network.bw.out': 'instance_network_interface',
-        'network.bw.in': 'instance_network_interface',
-        'network.floating': 'network',
-        'image': 'image',
-        'radosgw.usage': 'ceph_account',
-    },
-
-    'metrics': {
-        'vcpus': {
-            'resource': 'instance',
-            'unit': 'instance',
-            'factor': 1,
-            'aggregation_method': 'max',
-            'countable_unit': True,
-        },
-        'memory': {
-            'resource': 'instance',
-            'unit': 'instance',
-            'factor': 1,
-            'aggregation_method': 'max',
-            'countable_unit': True,
-        },
-        'cpu': {
-            'resource': 'instance',
-            'unit': 'instance',
-            'factor': 1,
-            'aggregation_method': 'max',
-            'countable_unit': True,
-        },
-        'disk.root.size': {
-            'resource': 'instance',
-            'unit': 'instance',
-            'factor': 1,
-            'aggregation_method': 'max',
-            'countable_unit': True,
-        },
-        'disk.ephemeral.size': {
-            'resource': 'instance',
-            'unit': 'instance',
-            'factor': 1,
-            'aggregation_method': 'max',
-            'countable_unit': True,
-        },
-        'image.size': {
-            'resource': 'image',
-            'unit': 'MiB',
-            'factor': 1 / 1048576,
-            'aggregation_method': 'max',
-        },
-        'image.download': {
-            'resource': 'image',
-            'unit': 'MiB',
-            'factor': 1 / 1048576,
-            'aggregation_method': 'max',
-        },
-        'image.serve': {
-            'resource': 'image',
-            'unit': 'MiB',
-            'factor': 1 / 1048576,
-            'aggregation_method': 'max',
-        },
-        'volume.size': {
-            'resource': 'volume',
-            'unit': 'GiB',
-            'factor': 1,
-            'aggregation_method': 'max',
-        },
-        'network.outgoing.bytes': {
-            'resource': 'instance_network_interface',
-            'unit': 'MB',
-            'factor': 1 / 1000000,
-            'aggregation_method': 'max',
-        },
-        'network.incoming.bytes': {
-            'resource': 'instance_network_interface',
-            'unit': 'MB',
-            'factor': 1 / 1000000,
-            'aggregation_method': 'max',
-        },
-        'ip.floating': {
-            'resource': 'network',
-            'unit': 'ip',
-            'factor': 1,
-            'aggregation_method': 'max',
-            'countable_unit': True,
-        },
-        'radosgw.objects.size': {
-            'resource': 'ceph_account',
-            'unit': 'GiB',
-            'factor': 1 / 1073741824,
-            'aggregation_method': 'max',
-        },
-    },
-}
diff --git a/cloudkitty/fetcher/__init__.py b/cloudkitty/fetcher/__init__.py
index 9db10ea5..8d9bf7a2 100644
--- a/cloudkitty/fetcher/__init__.py
+++ b/cloudkitty/fetcher/__init__.py
@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-# !/usr/bin/env python
 # Copyright 2015 Objectif Libre
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -21,13 +20,15 @@ import abc
 from oslo_config import cfg
 import six
 
+FETCHER_OPTS = 'fetcher'
+DEPRECATED_FETCHER_OPTS = 'tenant_fetcher'
 fetchers_opts = [
     cfg.StrOpt('backend',
                default='keystone',
-               help='Driver used to fetch tenant list.')
+               help='Driver used to fetch tenant list.',
+               deprecated_group=DEPRECATED_FETCHER_OPTS)
 ]
-
-cfg.CONF.register_opts(fetchers_opts, 'tenant_fetcher')
+cfg.CONF.register_opts(fetchers_opts, 'fetcher')
 
 
 @six.add_metaclass(abc.ABCMeta)
diff --git a/cloudkitty/fetcher/keystone.py b/cloudkitty/fetcher/keystone.py
index cac58a1b..15aa0812 100644
--- a/cloudkitty/fetcher/keystone.py
+++ b/cloudkitty/fetcher/keystone.py
@@ -61,18 +61,18 @@ class KeystoneFetcher(fetcher.BaseFetcher):
             session=self.session,
             auth_url=self.auth.auth_url)
 
-    def get_tenants(self, conf=None):
+    def get_tenants(self):
         keystone_version = discover.normalize_version_number(
             CONF.keystone_fetcher.keystone_version)
         auth_dispatch = {(3,): ('project', 'projects', 'list'),
                          (2,): ('tenant', 'tenants', 'roles_for_user')}
         for auth_version, auth_version_mapping in auth_dispatch.items():
             if discover.version_match(auth_version, keystone_version):
-                return self._do_get_tenants(auth_version_mapping, conf)
+                return self._do_get_tenants(auth_version_mapping)
         msg = "Keystone version you've specified is not supported"
         raise exceptions.VersionNotAvailable(msg)
 
-    def _do_get_tenants(self, auth_version_mapping, conf):
+    def _do_get_tenants(self, auth_version_mapping):
         tenant_attr, tenants_attr, role_func = auth_version_mapping
         tenant_list = getattr(self.admin_ks, tenants_attr).list()
         my_user_id = self.session.get_user_id()
@@ -82,8 +82,4 @@ class KeystoneFetcher(fetcher.BaseFetcher):
                    tenant_attr: tenant})
             if 'rating' not in [role.name for role in roles]:
                 tenant_list.remove(tenant)
-        if conf:
-            res = [{'tenant_id': tenant.id} for tenant in tenant_list]
-            for tenant in res:
-                tenant.update(conf)
-        return res
+        return [tenant.id for tenant in tenant_list]
diff --git a/cloudkitty/fetcher/source.py b/cloudkitty/fetcher/source.py
index 50a773fb..2dac20f9 100644
--- a/cloudkitty/fetcher/source.py
+++ b/cloudkitty/fetcher/source.py
@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-# !/usr/bin/env python
 # Copyright 2015 Objectif Libre
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -16,22 +15,25 @@
 #
 # @author: Martin CAMEY
 #
-import hashlib
+from oslo_config import cfg
 
 from cloudkitty import fetcher
 
 
+SOURCE_FETCHER_OPTS = 'source_fetcher'
+source_fetcher_opts = [
+    cfg.ListOpt('sources',
+                default=list(),
+                help='list of source identifiers'), ]
+
+cfg.CONF.register_opts(source_fetcher_opts, SOURCE_FETCHER_OPTS)
+CONF = cfg.CONF
+
+
 class SourceFetcher(fetcher.BaseFetcher):
     """Source projects fetcher."""
 
     name = 'source'
 
-    def get_projects(self, conf=None):
-        if conf:
-            tmp = hashlib.md5()
-            tmp.update(conf['name'])
-            conf['tenant_id'] = tmp.hexdigest()
-        return [conf]
-
-    def get_tenants(self, conf=None):
-        return self.get_projects(conf=conf)
+    def get_tenants(self):
+        return CONF.source_fetcher.sources
diff --git a/cloudkitty/orchestrator.py b/cloudkitty/orchestrator.py
index 92b8df3b..7357861c 100644
--- a/cloudkitty/orchestrator.py
+++ b/cloudkitty/orchestrator.py
@@ -41,7 +41,6 @@ eventlet.monkey_patch()
 LOG = logging.getLogger(__name__)
 
 CONF = cfg.CONF
-CONF.import_opt('backend', 'cloudkitty.fetcher', 'tenant_fetcher')
 
 orchestrator_opts = [
     cfg.StrOpt('coordination_url',
@@ -51,9 +50,9 @@ orchestrator_opts = [
 ]
 CONF.register_opts(orchestrator_opts, group='orchestrator')
 
-METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
+CONF.import_opt('backend', 'cloudkitty.fetcher', 'fetcher')
 
-FETCHERS_NAMESPACE = 'cloudkitty.tenant.fetchers'
+FETCHERS_NAMESPACE = 'cloudkitty.fetchers'
 PROCESSORS_NAMESPACE = 'cloudkitty.rating.processors'
 COLLECTORS_NAMESPACE = 'cloudkitty.collector.backends'
 STORAGES_NAMESPACE = 'cloudkitty.storage.backends'
@@ -153,13 +152,13 @@ class APIWorker(BaseWorker):
 
 
 class Worker(BaseWorker):
-    def __init__(self, collector, storage, tenant):
+    def __init__(self, collector, storage, tenant_id):
         self._collector = collector
         self._storage = storage
-        self._period = tenant['period']
-        self._wait_time = tenant['wait_periods'] * self._period
-        self._tenant_id = tenant['tenant_id']
-        self.conf = tenant
+        self._period = CONF.collect.period
+        self._wait_time = CONF.collect.wait_periods * self._period
+        self._tenant_id = tenant_id
+        self._conf = ck_utils.load_conf(CONF.collect.metrics_conf)
 
         super(Worker, self).__init__(self._tenant_id)
 
@@ -182,7 +181,7 @@ class Worker(BaseWorker):
         timestamp = self._storage.get_state(self._tenant_id)
         return ck_utils.check_time_state(timestamp,
                                          self._period,
-                                         self._wait_time)
+                                         CONF.collect.wait_periods)
 
     def run(self):
         while True:
@@ -190,7 +189,7 @@ class Worker(BaseWorker):
             if not timestamp:
                 break
 
-            metrics = list(self.conf['metrics'].keys())
+            metrics = list(self._conf['metrics'].keys())
 
             for metric in metrics:
                 try:
@@ -225,8 +224,8 @@ class Orchestrator(object):
     def __init__(self):
         self.fetcher = driver.DriverManager(
             FETCHERS_NAMESPACE,
-            METRICS_CONF['fetcher'],
-            invoke_on_load=True
+            CONF.fetcher.backend,
+            invoke_on_load=True,
         ).driver
 
         transformers = transformer.get_transformers()
@@ -258,11 +257,11 @@ class Orchestrator(object):
         self.server = messaging.get_server(target, endpoints)
         self.server.start()
 
-    def _check_state(self, tenant_id, period, wait_time):
+    def _check_state(self, tenant_id):
         timestamp = self.storage.get_state(tenant_id)
         return ck_utils.check_time_state(timestamp,
-                                         period,
-                                         wait_time)
+                                         CONF.collect.period,
+                                         CONF.collect.wait_periods)
 
     def process_messages(self):
         # TODO(sheeprine): Code kept to handle threading and asynchronous
@@ -273,36 +272,31 @@ class Orchestrator(object):
 
     def process(self):
         while True:
-            self.tenants = self.fetcher.get_tenants(METRICS_CONF)
+            self.tenants = self.fetcher.get_tenants()
             random.shuffle(self.tenants)
             LOG.info('Tenants loaded for fetcher %s', self.fetcher.name)
 
-            for tenant in self.tenants:
-                lock = self._lock(tenant['tenant_id'])
+            for tenant_id in self.tenants:
+
+                lock = self._lock(tenant_id)
                 if lock.acquire(blocking=False):
-                    state = self._check_state(
-                        tenant['tenant_id'],
-                        tenant['period'],
-                        tenant['wait_periods'],
-                    )
-                    if not state:
-                        self.tenants.remove(tenant)
-                    else:
+                    state = self._check_state(tenant_id)
+                    if state:
                         worker = Worker(
                             self.collector,
                             self.storage,
-                            tenant,
+                            tenant_id,
                         )
-
                         worker.run()
                     lock.release()
+
                 self.coord.heartbeat()
 
                 # NOTE(sheeprine): Slow down looping if all tenants are
                 # being processed
                 eventlet.sleep(1)
             # FIXME(sheeprine): We may cause a drift here
-            eventlet.sleep(tenant['period'])
+            eventlet.sleep(CONF.collect.period)
 
     def terminate(self):
         self.coord.stop()
diff --git a/cloudkitty/storage/__init__.py b/cloudkitty/storage/__init__.py
index 3c009197..4d8a3c20 100644
--- a/cloudkitty/storage/__init__.py
+++ b/cloudkitty/storage/__init__.py
@@ -25,21 +25,28 @@ from stevedore import driver
 from cloudkitty import utils as ck_utils
 
 
+storage_opts = [
+    cfg.StrOpt('backend',
+               default='sqlalchemy',
+               help='Name of the storage backend driver.')
+]
+
 LOG = logging.getLogger(__name__)
 
 CONF = cfg.CONF
 
-# NOTE(mc): This hack is possible because only
-# one OpenStack configuration is allowed.
-METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
+CONF.import_opt('period', 'cloudkitty.collector', 'collect')
+
+CONF.register_opts(storage_opts, 'storage')
 
 STORAGES_NAMESPACE = 'cloudkitty.storage.backends'
 
 
-def get_storage():
+def get_storage(**kwargs):
     storage_args = {
-        'period': METRICS_CONF.get('period', 3600),
+        'period': CONF.collect.period,
     }
+    storage_args.update(kwargs)
     backend = driver.DriverManager(
         STORAGES_NAMESPACE,
         cfg.CONF.storage.backend,
diff --git a/cloudkitty/storage/hybrid/backends/gnocchi.py b/cloudkitty/storage/hybrid/backends/gnocchi.py
index e6e5888a..1604f01e 100644
--- a/cloudkitty/storage/hybrid/backends/gnocchi.py
+++ b/cloudkitty/storage/hybrid/backends/gnocchi.py
@@ -27,8 +27,8 @@ from oslo_log import log as logging
 from oslo_utils import uuidutils
 import six
 
+from cloudkitty.collector import validate_conf
 from cloudkitty.storage.hybrid.backends import BaseHybridBackend
-from cloudkitty.transformer import gnocchi as gtransformer
 import cloudkitty.utils as ck_utils
 
 
@@ -36,10 +36,6 @@ LOG = logging.getLogger(__name__)
 
 CONF = cfg.CONF
 
-# NOTE(mc): This hack is possible because only
-# one OpenStack configuration is allowed.
-METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
-
 CONF.import_opt('period', 'cloudkitty.collector', 'collect')
 
 GNOCCHI_STORAGE_OPTS = 'storage_gnocchi'
@@ -53,7 +49,7 @@ gnocchi_storage_opts = [
     # The archive policy definition MUST include the collect period granularity
     cfg.StrOpt('archive_policy_definition',
                default='[{"granularity": '
-                       + six.text_type(METRICS_CONF.get('period', 3600)) +
+                       + six.text_type(CONF.collect.period) +
                        ', "timespan": "90 days"}, '
                        '{"granularity": 86400, "timespan": "360 days"}, '
                        '{"granularity": 2592000, "timespan": "1800 days"}]',
@@ -68,6 +64,7 @@ ks_loading.register_auth_conf_options(
     GNOCCHI_STORAGE_OPTS)
 
 RESOURCE_TYPE_NAME_ROOT = 'rating_service_'
+METADATA_NAME_ROOT = 'ckmeta_'
 
 
 class DecimalJSONEncoder(json.JSONEncoder):
@@ -92,34 +89,27 @@ class GnocchiStorage(BaseHybridBackend):
 
     """
 
-    # NOTE(lukapeschke): List taken directly from gnocchi code
-    invalid_attribute_names = [
-        "id", "type", "metrics",
-        "revision", "revision_start", "revision_end",
-        "started_at", "ended_at",
-        "user_id", "project_id",
-        "created_by_user_id", "created_by_project_id", "get_metric",
-        "creator",
-    ]
+    groupby_keys = ['res_type', 'tenant_id']
+    groupby_values = ['type', 'project_id']
 
     def _init_resource_types(self):
-        transformer = gtransformer.GnocchiTransformer()
-        for metric in list(self.conf['metrics'].keys()):
+        for metric_name, metric in self.conf.items():
             metric_dict = dict()
             metric_dict['attributes'] = list()
-            for attribute in transformer.get_metadata(metric):
-                if attribute not in self.invalid_attribute_names:
-                    metric_dict['attributes'].append(attribute)
-            metric_dict['required_attributes'] = [
-                'resource_id',
-                'unit',
-            ]
-            metric_dict['name'] = RESOURCE_TYPE_NAME_ROOT + metric
-            metric_dict['qty_metric'] = 1
-            if self.conf['metrics'][metric].get('countable_unit'):
-                resource = self.conf['metrics'][metric]['resource']
-                metric_dict['qty_metric'] = resource
-            self._resource_type_data[metric] = metric_dict
+            for attribute in metric.get('metadata', {}):
+                metric_dict['attributes'].append(
+                    METADATA_NAME_ROOT + attribute)
+            metric_dict['required_attributes'] = ['unit', 'resource_id']
+            for attribute in metric['groupby']:
+                metric_dict['required_attributes'].append(
+                    METADATA_NAME_ROOT + attribute)
+
+            metric_dict['name'] = RESOURCE_TYPE_NAME_ROOT + metric['alt_name']
+            if metric['mutate'] == 'NUMBOOL':
+                metric_dict['qty_metric'] = 1
+            else:
+                metric_dict['qty_metric'] = metric_name
+            self._resource_type_data[metric['alt_name']] = metric_dict
 
     def _get_res_type_dict(self, res_type):
         res_type_data = self._resource_type_data.get(res_type, None)
@@ -148,16 +138,19 @@ class GnocchiStorage(BaseHybridBackend):
                 "Unknown resource type '{}'".format(res_type))
 
         res_dict = {
-            'id': data['resource_id'],
-            'resource_id': data['resource_id'],
+            'id': data['id'],
+            'resource_id': data['id'],
             'project_id': tenant_id,
-            'user_id': data['user_id'],
+            'user_id': 'cloudkitty',
             'unit': data['unit'],
         }
-        for attr in res_type_data['attributes']:
-            res_dict[attr] = data.get(attr, None) or 'None'
-            if isinstance(res_dict[attr], decimal.Decimal):
-                res_dict[attr] = float(res_dict[attr])
+        for key in ['attributes', 'required_attributes']:
+            for attr in res_type_data[key]:
+                if METADATA_NAME_ROOT in attr:
+                    res_dict[attr] = data.get(
+                        attr.replace(METADATA_NAME_ROOT, ''), None) or ''
+                    if isinstance(res_dict[attr], decimal.Decimal):
+                        res_dict[attr] = float(res_dict[attr])
 
         created_metrics = [
             self._conn.metric.create({
@@ -224,7 +217,9 @@ class GnocchiStorage(BaseHybridBackend):
 
     def __init__(self, **kwargs):
         super(GnocchiStorage, self).__init__(**kwargs)
-        self.conf = kwargs['conf'] if 'conf' in kwargs else METRICS_CONF
+        conf = kwargs.get('conf') or ck_utils.load_conf(
+            CONF.collect.metrics_conf)
+        self.conf = validate_conf(conf)
         self.auth = ks_loading.load_auth_from_conf_options(
             CONF,
             GNOCCHI_STORAGE_OPTS)
@@ -241,9 +236,7 @@ class GnocchiStorage(BaseHybridBackend):
             CONF.storage_gnocchi.archive_policy_name)
         self._archive_policy_definition = json.loads(
             CONF.storage_gnocchi.archive_policy_definition)
-        self._period = self.conf['period']
-        if "period" in kwargs:
-            self._period = kwargs["period"]
+        self._period = kwargs.get('period') or CONF.collect.period
         self._measurements = dict()
         self._resource_type_data = dict()
         self._init_resource_types()
@@ -288,21 +281,57 @@ class GnocchiStorage(BaseHybridBackend):
     def get_total(self, begin=None, end=None, tenant_id=None,
                   service=None, groupby=None):
         # Query can't be None if we don't specify a resource_id
-        query = {}
+        query = {'and': [{
+            'like': {'type': RESOURCE_TYPE_NAME_ROOT + '%'},
+        }]}
         if tenant_id:
-            query['='] = {"project_id": tenant_id}
-        measures = self._conn.metric.aggregation(
-            metrics='price', query=query,
-            start=begin, stop=end,
-            aggregation='sum',
-            granularity=self._period,
-            needed_overlap=0)
-        rate = sum(measure[2] for measure in measures) if len(measures) else 0
-        return [{
-            'begin': begin,
-            'end': end,
-            'rate': rate,
-        }]
+            query['and'].append({'=': {'project_id': tenant_id}})
+
+        gb = []
+        if groupby:
+            for elem in groupby.split(','):
+                if elem in self.groupby_keys:
+                    gb.append(self.groupby_values[
+                        self.groupby_keys.index(elem)])
+        # Setting gb to None instead of an empty list
+        gb = gb if len(gb) > 0 else None
+
+        # build aggregration operation
+        op = ['aggregate', 'sum', ['metric', 'price', 'sum']]
+
+        try:
+            aggregates = self._conn.aggregates.fetch(
+                op,
+                start=begin,
+                stop=end,
+                groupby=gb,
+                search=query)
+        # No 'price' metric found
+        except gexceptions.BadRequest:
+            return [dict(begin=begin, end=end, rate=0)]
+
+        # In case no group_by was specified
+        if not isinstance(aggregates, list):
+            aggregates = [aggregates]
+        total_list = list()
+        for aggregate in aggregates:
+            if groupby:
+                measures = aggregate['measures']['measures']['aggregated']
+            else:
+                measures = aggregate['measures']['aggregated']
+            if len(measures) > 0:
+                rate = sum(measure[2] for measure in measures
+                           if (measure[1] == self._period))
+                total = dict(begin=begin, end=end, rate=rate)
+                if gb:
+                    for value in gb:
+                        key = self.groupby_keys[
+                            self.groupby_values.index(value)]
+                        total[key] = aggregate['group'][value].replace(
+                            RESOURCE_TYPE_NAME_ROOT, '')
+                total_list.append(total)
+
+        return total_list
 
     def _append_measurements(self, resource, data, tenant_id):
         if not self._measurements.get(tenant_id, None):
@@ -322,7 +351,7 @@ class GnocchiStorage(BaseHybridBackend):
 
     def append_time_frame(self, res_type, frame, tenant_id):
         flat_frame = ck_utils.flat_dict(frame)
-        resource = self._find_resource(res_type, flat_frame['resource_id'])
+        resource = self._find_resource(res_type, flat_frame['id'])
         if not resource:
             resource = self._create_resource(res_type, tenant_id, flat_frame)
         self._append_measurements(resource, flat_frame, tenant_id)
@@ -441,7 +470,8 @@ class GnocchiStorage(BaseHybridBackend):
                         resource_type, resource_measures['group']['id'])
                     if not resource:
                         continue
-                    desc = {a: resource.get(a, None) for a in attributes}
+                    desc = {attr.replace(METADATA_NAME_ROOT, ''):
+                            resource.get(attr, None) for attr in attributes}
                 formatted_frame = self._format_frame(
                     resource_type, resource, desc, measure, tenant_id)
                 output.append(formatted_frame)
diff --git a/cloudkitty/tests/gabbi/fixtures.py b/cloudkitty/tests/gabbi/fixtures.py
index d737d866..88ab7166 100644
--- a/cloudkitty/tests/gabbi/fixtures.py
+++ b/cloudkitty/tests/gabbi/fixtures.py
@@ -43,6 +43,7 @@ from cloudkitty import rating
 from cloudkitty import storage
 from cloudkitty.storage.sqlalchemy import models
 from cloudkitty import tests
+from cloudkitty.tests import test_utils
 from cloudkitty import utils as ck_utils
 
 
@@ -278,7 +279,7 @@ class BaseStorageDataFixture(fixture.GabbiFixture):
                 "begin": begin,
                 "end": end},
             "usage": {
-                "compute": [
+                "cpu": [
                     {
                         "desc": {
                             "dummy": True,
@@ -292,7 +293,7 @@ class BaseStorageDataFixture(fixture.GabbiFixture):
                 "begin": begin,
                 "end": end},
             "usage": {
-                "image": [
+                "image.size": [
                     {
                         "desc": {
                             "dummy": True,
@@ -313,7 +314,7 @@ class BaseStorageDataFixture(fixture.GabbiFixture):
             return_value=dict())
         with auth:
             with session:
-                self.storage = storage.get_storage()
+                self.storage = storage.get_storage(conf=test_utils.load_conf())
         self.storage.init()
         self.initialize_data()
 
@@ -391,14 +392,14 @@ class MetricsConfFixture(fixture.GabbiFixture):
     """Inject Metrics configuration mock to the get_metrics_conf() function"""
 
     def start_fixture(self):
-        self._original_function = ck_utils.get_metrics_conf
-        ck_utils.get_metrics_conf = mock.Mock(
+        self._original_function = ck_utils.load_conf
+        ck_utils.load_conf = mock.Mock(
             return_value=tests.samples.METRICS_CONF,
         )
 
     def stop_fixture(self):
         """Remove the get_metrics_conf() monkeypatch."""
-        ck_utils.get_metrics_conf = self._original_function
+        ck_utils.load_conf = self._original_function
 
 
 def setup_app():
diff --git a/cloudkitty/tests/gabbi/gabbits/v1-billing.yaml b/cloudkitty/tests/gabbi/gabbits/v1-billing.yaml
index aee01992..84a96915 100644
--- a/cloudkitty/tests/gabbi/gabbits/v1-billing.yaml
+++ b/cloudkitty/tests/gabbi/gabbits/v1-billing.yaml
@@ -123,7 +123,7 @@ tests:
       x-roles: admin
     data:
       resources:
-        - service: "compute"
+        - service: "cpu"
           volume: "1.0"
           desc:
             test: 1
diff --git a/cloudkitty/tests/gabbi/gabbits/v1-info.yaml b/cloudkitty/tests/gabbi/gabbits/v1-info.yaml
index 1643bb72..4623e982 100644
--- a/cloudkitty/tests/gabbi/gabbits/v1-info.yaml
+++ b/cloudkitty/tests/gabbi/gabbits/v1-info.yaml
@@ -7,20 +7,9 @@ tests:
     url: /v1/info/config
     status: 200
     response_json_paths:
-      $.collector: gnocchi
-      $.window: 1800
-      $.wait_periods: 2
-      $.period: 3600
-
-      $.metrics.`len`: 13
-      $.metrics.vcpus.unit: instance
-      $.metrics.memory.unit: instance
-      $.metrics.cpu.unit: instance
-      $.metrics['disk.root.size'].unit: instance
-      $.metrics['disk.ephemeral.size'].unit: instance
+      $.metrics.`len`: 7
+      $.metrics['cpu'].unit: instance
       $.metrics['image.size'].unit: MiB
-      $.metrics['image.download'].unit: MiB
-      $.metrics['image.serve'].unit: MiB
       $.metrics['volume.size'].unit: GiB
       $.metrics['network.incoming.bytes'].unit: MB
       $.metrics['network.outgoing.bytes'].unit: MB
@@ -31,38 +20,25 @@ tests:
     url: /v1/info/metrics
     status: 200
     response_json_paths:
-      $.metrics.`len`: 13
-      $.metrics[/metric_id][0].metric_id: cpu
-      $.metrics[/metric_id][0].unit: instance
-      $.metrics[/metric_id][1].metric_id: disk.ephemeral.size
+      $.metrics.`len`: 7
+      $.metrics[/metric_id][0].metric_id: image.size
+      $.metrics[/metric_id][0].unit: MiB
+      $.metrics[/metric_id][1].metric_id: instance
       $.metrics[/metric_id][1].unit: instance
-      $.metrics[/metric_id][2].metric_id: disk.root.size
-      $.metrics[/metric_id][2].unit: instance
-      $.metrics[/metric_id][3].metric_id: image.download
-      $.metrics[/metric_id][3].unit: MiB
-      $.metrics[/metric_id][4].metric_id: image.serve
-      $.metrics[/metric_id][4].unit: MiB
-      $.metrics[/metric_id][5].metric_id: image.size
-      $.metrics[/metric_id][5].unit: MiB
-      $.metrics[/metric_id][6].metric_id: ip.floating
-      $.metrics[/metric_id][6].unit: ip
-      $.metrics[/metric_id][7].metric_id: memory
-      $.metrics[/metric_id][7].unit: instance
-      $.metrics[/metric_id][8].metric_id: network.incoming.bytes
-      $.metrics[/metric_id][8].unit: MB
-      $.metrics[/metric_id][9].metric_id: network.outgoing.bytes
-      $.metrics[/metric_id][9].unit: MB
-      $.metrics[/metric_id][10].metric_id: radosgw.objects.size
-      $.metrics[/metric_id][10].unit: GiB
-      $.metrics[/metric_id][11].metric_id: vcpus
-      $.metrics[/metric_id][11].unit: instance
-      $.metrics[/metric_id][12].metric_id: volume.size
-      $.metrics[/metric_id][12].unit: GiB
+      $.metrics[/metric_id][2].metric_id: ip.floating
+      $.metrics[/metric_id][2].unit: ip
+      $.metrics[/metric_id][3].metric_id: network.incoming.bytes
+      $.metrics[/metric_id][3].unit: MB
+      $.metrics[/metric_id][4].metric_id: network.outgoing.bytes
+      $.metrics[/metric_id][4].unit: MB
+      $.metrics[/metric_id][5].metric_id: radosgw.objects.size
+      $.metrics[/metric_id][5].unit: GiB
+      $.metrics[/metric_id][6].metric_id: volume.size
+      $.metrics[/metric_id][6].unit: GiB
 
   - name: get cpu metric info
-    url: /v1/info/metrics/cpu
+    url: /v1/info/metrics/instance
     status: 200
     response_json_paths:
-      $.metric_id: cpu
+      $.metric_id: instance
       $.unit: instance
-      $.metadata.`len`: 4
diff --git a/cloudkitty/tests/gabbi/gabbits/v1-rating.yaml b/cloudkitty/tests/gabbi/gabbits/v1-rating.yaml
index dfe8062b..433e04a0 100644
--- a/cloudkitty/tests/gabbi/gabbits/v1-rating.yaml
+++ b/cloudkitty/tests/gabbi/gabbits/v1-rating.yaml
@@ -123,7 +123,7 @@ tests:
       x-roles: admin
     data:
       resources:
-        - service: "compute"
+        - service: "cpu"
           volume: "1.0"
           desc:
             test: 1
diff --git a/cloudkitty/tests/gabbi/gabbits/v1-report.yaml b/cloudkitty/tests/gabbi/gabbits/v1-report.yaml
index c473b67d..948bd97b 100644
--- a/cloudkitty/tests/gabbi/gabbits/v1-report.yaml
+++ b/cloudkitty/tests/gabbi/gabbits/v1-report.yaml
@@ -73,7 +73,7 @@ tests:
     query_parameters:
       begin: "2015-01-01T00:00:00"
       end: "2015-02-04T00:00:00"
-      service: "compute"
+      service: "cpu"
     status: 200
     response_strings:
       - "110.971"
@@ -83,7 +83,7 @@ tests:
     query_parameters:
       begin: "2015-01-01T00:00:00"
       end: "2015-02-04T00:00:00"
-      service: "image"
+      service: "image.size"
     status: 200
     response_strings:
       - "10.043"
@@ -94,7 +94,7 @@ tests:
       begin: "2015-01-01T00:00:00"
       end: "2015-02-04T00:00:00"
       tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
-      service: "compute"
+      service: "cpu"
     status: 200
     response_strings:
       - "37.436"
@@ -133,19 +133,19 @@ tests:
     query_parameters:
       begin: "2015-01-01T00:00:00"
       end: "2015-02-04T00:00:00"
-      service: "compute"
+      service: "cpu"
       groupby: "tenant_id"
     status: 200
     response_json_paths:
       $.summary.`len`: 2
       $.summary[0].rate: "37.436"
       $.summary[0].tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
-      $.summary[0].res_type: "compute"
+      $.summary[0].res_type: "cpu"
       $.summary[0].begin: "2015-01-01T00:00:00"
       $.summary[0].end: "2015-02-04T00:00:00"
       $.summary[1].rate: "73.535"
       $.summary[1].tenant_id: "8f82cc70-e50c-466e-8624-24bdea811375"
-      $.summary[1].res_type: "compute"
+      $.summary[1].res_type: "cpu"
       $.summary[1].begin: "2015-01-01T00:00:00"
       $.summary[1].end: "2015-02-04T00:00:00"
 
@@ -159,12 +159,12 @@ tests:
     response_json_paths:
       $.summary.`len`: 2
       $.summary[0].rate: "10.043"
-      $.summary[0].res_type: "image"
+      $.summary[0].res_type: "image.size"
       $.summary[0].tenant_id: "ALL"
       $.summary[0].begin: "2015-01-01T00:00:00"
       $.summary[0].end: "2015-02-04T00:00:00"
       $.summary[1].rate: "110.971"
-      $.summary[1].res_type: "compute"
+      $.summary[1].res_type: "cpu"
       $.summary[1].tenant_id: "ALL"
       $.summary[1].begin: "2015-01-01T00:00:00"
       $.summary[1].end: "2015-02-04T00:00:00"
@@ -181,12 +181,12 @@ tests:
       $.summary.`len`: 2
       $.summary[0].rate: "6.655"
       $.summary[0].tenant_id: "8f82cc70-e50c-466e-8624-24bdea811375"
-      $.summary[0].res_type: "image"
+      $.summary[0].res_type: "image.size"
       $.summary[0].begin: "2015-01-01T00:00:00"
       $.summary[0].end: "2015-02-04T00:00:00"
       $.summary[1].rate: "73.535"
       $.summary[1].tenant_id: "8f82cc70-e50c-466e-8624-24bdea811375"
-      $.summary[1].res_type: "compute"
+      $.summary[1].res_type: "cpu"
       $.summary[1].begin: "2015-01-01T00:00:00"
       $.summary[1].end: "2015-02-04T00:00:00"
 
@@ -200,22 +200,22 @@ tests:
     response_json_paths:
       $.summary.`len`: 4
       $.summary[0].rate: "3.388"
-      $.summary[0].res_type: "image"
+      $.summary[0].res_type: "image.size"
       $.summary[0].tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
       $.summary[0].begin: "2015-01-01T00:00:00"
       $.summary[0].end: "2015-02-04T00:00:00"
       $.summary[1].rate: "6.655"
-      $.summary[1].res_type: "image"
+      $.summary[1].res_type: "image.size"
       $.summary[1].tenant_id: "8f82cc70-e50c-466e-8624-24bdea811375"
       $.summary[1].begin: "2015-01-01T00:00:00"
       $.summary[1].end: "2015-02-04T00:00:00"
       $.summary[2].rate: "37.436"
-      $.summary[2].res_type: "compute"
+      $.summary[2].res_type: "cpu"
       $.summary[2].tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
       $.summary[2].begin: "2015-01-01T00:00:00"
       $.summary[2].end: "2015-02-04T00:00:00"
       $.summary[3].rate: "73.535"
-      $.summary[3].res_type: "compute"
+      $.summary[3].res_type: "cpu"
       $.summary[3].tenant_id: "8f82cc70-e50c-466e-8624-24bdea811375"
       $.summary[3].begin: "2015-01-01T00:00:00"
       $.summary[3].end: "2015-02-04T00:00:00"
diff --git a/cloudkitty/tests/gabbi/gabbits/v1-storage.yaml b/cloudkitty/tests/gabbi/gabbits/v1-storage.yaml
index 442a6e7f..373a6b41 100644
--- a/cloudkitty/tests/gabbi/gabbits/v1-storage.yaml
+++ b/cloudkitty/tests/gabbi/gabbits/v1-storage.yaml
@@ -74,7 +74,7 @@ tests:
       $.dataframes[0].resources.`len`: 1
       $.dataframes[0].resources[0].volume: "1"
       $.dataframes[0].resources[0].rating: "1.337"
-      $.dataframes[0].resources[0].service: "compute"
+      $.dataframes[0].resources[0].service: "cpu"
       $.dataframes[0].resources[0].desc.dummy: true
       $.dataframes[0].resources[0].desc.fake_meta: 1.0
       $.dataframes[1].tenant_id: "8f82cc70-e50c-466e-8624-24bdea811375"
@@ -83,7 +83,7 @@ tests:
       $.dataframes[1].resources.`len`: 1
       $.dataframes[1].resources[0].volume: "1"
       $.dataframes[1].resources[0].rating: "0.121"
-      $.dataframes[1].resources[0].service: "image"
+      $.dataframes[1].resources[0].service: "image.size"
       $.dataframes[1].resources[0].desc.dummy: true
       $.dataframes[1].resources[0].desc.fake_meta: 1.0
 
@@ -102,7 +102,7 @@ tests:
       $.dataframes[0].resources.`len`: 1
       $.dataframes[0].resources[0].volume: "1"
       $.dataframes[0].resources[0].rating: "1.337"
-      $.dataframes[0].resources[0].service: "compute"
+      $.dataframes[0].resources[0].service: "cpu"
       $.dataframes[0].resources[0].desc.dummy: true
       $.dataframes[0].resources[0].desc.fake_meta: 1.0
       $.dataframes[1].tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
@@ -111,7 +111,7 @@ tests:
       $.dataframes[1].resources.`len`: 1
       $.dataframes[1].resources[0].volume: "1"
       $.dataframes[1].resources[0].rating: "0.121"
-      $.dataframes[1].resources[0].service: "image"
+      $.dataframes[1].resources[0].service: "image.size"
       $.dataframes[1].resources[0].desc.dummy: true
       $.dataframes[1].resources[0].desc.fake_meta: 1.0
 
@@ -129,7 +129,7 @@ tests:
       $.dataframes[0].resources.`len`: 1
       $.dataframes[0].resources[0].volume: "1"
       $.dataframes[0].resources[0].rating: "1.337"
-      $.dataframes[0].resources[0].service: "compute"
+      $.dataframes[0].resources[0].service: "cpu"
       $.dataframes[0].resources[0].desc.dummy: true
       $.dataframes[0].resources[0].desc.fake_meta: 1.0
       $.dataframes[1].tenant_id: "8f82cc70-e50c-466e-8624-24bdea811375"
@@ -138,7 +138,7 @@ tests:
       $.dataframes[1].resources.`len`: 1
       $.dataframes[1].resources[0].volume: "1"
       $.dataframes[1].resources[0].rating: "0.121"
-      $.dataframes[1].resources[0].service: "image"
+      $.dataframes[1].resources[0].service: "image.size"
       $.dataframes[1].resources[0].desc.dummy: true
       $.dataframes[1].resources[0].desc.fake_meta: 1.0
       $.dataframes[2].tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
@@ -147,7 +147,7 @@ tests:
       $.dataframes[2].resources.`len`: 1
       $.dataframes[2].resources[0].volume: "1"
       $.dataframes[2].resources[0].rating: "1.337"
-      $.dataframes[2].resources[0].service: "compute"
+      $.dataframes[2].resources[0].service: "cpu"
       $.dataframes[2].resources[0].desc.dummy: true
       $.dataframes[2].resources[0].desc.fake_meta: 1.0
       $.dataframes[3].tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
@@ -156,16 +156,16 @@ tests:
       $.dataframes[3].resources.`len`: 1
       $.dataframes[3].resources[0].volume: "1"
       $.dataframes[3].resources[0].rating: "0.121"
-      $.dataframes[3].resources[0].service: "image"
+      $.dataframes[3].resources[0].service: "image.size"
       $.dataframes[3].resources[0].desc.dummy: true
       $.dataframes[3].resources[0].desc.fake_meta: 1.0
 
-  - name: fetch data filtering on compute service and tenant
+  - name: fetch data filtering on cpu service and tenant
     url: /v1/storage/dataframes
     query_parameters:
       begin: "2015-01-04T13:00:00"
       end: "2015-01-04T14:00:00"
-      resource_type: "compute"
+      resource_type: "cpu"
       tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
     status: 200
     response_json_paths:
@@ -176,7 +176,7 @@ tests:
       $.dataframes[0].resources.`len`: 1
       $.dataframes[0].resources[0].volume: "1"
       $.dataframes[0].resources[0].rating: "1.337"
-      $.dataframes[0].resources[0].service: "compute"
+      $.dataframes[0].resources[0].service: "cpu"
       $.dataframes[0].resources[0].desc.dummy: true
       $.dataframes[0].resources[0].desc.fake_meta: 1.0
 
@@ -185,7 +185,7 @@ tests:
     query_parameters:
       begin: "2015-01-04T13:00:00"
       end: "2015-01-04T14:00:00"
-      resource_type: "image"
+      resource_type: "image.size"
       tenant_id: "7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
     status: 200
     response_json_paths:
@@ -196,7 +196,7 @@ tests:
       $.dataframes[0].resources.`len`: 1
       $.dataframes[0].resources[0].volume: "1"
       $.dataframes[0].resources[0].rating: "0.121"
-      $.dataframes[0].resources[0].service: "image"
+      $.dataframes[0].resources[0].service: "image.size"
       $.dataframes[0].resources[0].desc.dummy: true
       $.dataframes[0].resources[0].desc.fake_meta: 1.0
 
diff --git a/cloudkitty/tests/gabbi/rating/hash/gabbits/hash-errors.yaml b/cloudkitty/tests/gabbi/rating/hash/gabbits/hash-errors.yaml
index e3561a67..d34e8383 100644
--- a/cloudkitty/tests/gabbi/rating/hash/gabbits/hash-errors.yaml
+++ b/cloudkitty/tests/gabbi/rating/hash/gabbits/hash-errors.yaml
@@ -40,10 +40,10 @@ tests:
         content-type: application/json
         x-roles: admin
     data:
-      name: "compute"
+      name: "cpu"
     status: 201
     response_json_paths:
-      $.name: "compute"
+      $.name: "cpu"
     response_store_environ:
       hash_error_service_id: $.service_id
 
@@ -54,10 +54,10 @@ tests:
       content-type: application/json
       x-roles: admin
     data:
-      name: "compute"
+      name: "cpu"
     status: 409
     response_strings:
-      - "Service compute already exists (UUID: $RESPONSE['$.service_id'])"
+      - "Service cpu already exists (UUID: $RESPONSE['$.service_id'])"
 
   - name: create a service mapping with an invalid type
     url: /v1/rating/module_config/hashmap/mappings
diff --git a/cloudkitty/tests/gabbi/rating/hash/gabbits/hash-location.yaml b/cloudkitty/tests/gabbi/rating/hash/gabbits/hash-location.yaml
index b2bee901..a94545a1 100644
--- a/cloudkitty/tests/gabbi/rating/hash/gabbits/hash-location.yaml
+++ b/cloudkitty/tests/gabbi/rating/hash/gabbits/hash-location.yaml
@@ -11,11 +11,11 @@ tests:
         content-type: application/json
         x-roles: admin
     data:
-      name: "compute"
+      name: "cpu"
     status: 201
     response_json_paths:
       $.service_id: "6c1b8a30-797f-4b7e-ad66-9879b79059fb"
-      $.name: "compute"
+      $.name: "cpu"
     response_headers:
       location: $SCHEME://$NETLOC/v1/rating/module_config/hashmap/services/6c1b8a30-797f-4b7e-ad66-9879b79059fb
 
diff --git a/cloudkitty/tests/gabbi/rating/hash/gabbits/hash.yaml b/cloudkitty/tests/gabbi/rating/hash/gabbits/hash.yaml
index e060c056..f54cf84a 100644
--- a/cloudkitty/tests/gabbi/rating/hash/gabbits/hash.yaml
+++ b/cloudkitty/tests/gabbi/rating/hash/gabbits/hash.yaml
@@ -20,10 +20,10 @@ tests:
         content-type: application/json
         x-roles: admin
     data:
-      name: "compute"
+      name: "cpu"
     status: 201
     response_json_paths:
-      $.name: "compute"
+      $.name: "cpu"
     response_store_environ:
       hash_service_id: $.service_id
 
@@ -32,7 +32,7 @@ tests:
     status: 200
     response_json_paths:
       $.service_id: $RESPONSE['$.service_id']
-      $.name: "compute"
+      $.name: "cpu"
 
   - name: create a flat service mapping
     url: /v1/rating/module_config/hashmap/mappings
@@ -60,7 +60,7 @@ tests:
     status: 200
     response_json_paths:
       $.services.`len`: 1
-      $.services[0].name: "compute"
+      $.services[0].name: "cpu"
 
   - name: create a rate service mapping
     url: /v1/rating/module_config/hashmap/mappings
diff --git a/cloudkitty/tests/samples.py b/cloudkitty/tests/samples.py
index 7299db33..9ff81263 100644
--- a/cloudkitty/tests/samples.py
+++ b/cloudkitty/tests/samples.py
@@ -18,7 +18,6 @@
 import copy
 import decimal
 
-from cloudkitty.default_metrics_conf import DEFAULT_METRICS_CONF
 from cloudkitty import utils as ck_utils
 
 TENANT = 'f266f30b11f246b589fd266f85eeec39'
@@ -37,6 +36,7 @@ COMPUTE_METADATA = {
     'flavor': 'm1.nano',
     'image_id': 'f5600101-8fa2-4864-899e-ebcb7ed6b568',
     'instance_id': '26c084e1-b8f1-4cbc-a7ec-e8b356788a17',
+    'id': '1558f911-b55a-4fd2-9173-c8f1f23e5639',
     'resource_id': '1558f911-b55a-4fd2-9173-c8f1f23e5639',
     'memory': '64',
     'metadata': {
@@ -50,6 +50,7 @@ COMPUTE_METADATA = {
 IMAGE_METADATA = {
     'checksum': '836c69cbcd1dc4f225daedbab6edc7c7',
     'resource_id': '7b5b73f2-9181-4307-a710-b1aa6472526d',
+    'id': '7b5b73f2-9181-4307-a710-b1aa6472526d',
     'container_format': 'aki',
     'created_at': '2014-06-04T16:26:01',
     'deleted': 'False',
@@ -75,7 +76,7 @@ SECOND_PERIOD = {
 COLLECTED_DATA = [{
     'period': FIRST_PERIOD,
     'usage': {
-        'cpu': [{
+        'instance': [{
             'desc': COMPUTE_METADATA,
             'vol': {
                 'qty': decimal.Decimal(1.0),
@@ -88,22 +89,134 @@ COLLECTED_DATA = [{
     }}, {
     'period': SECOND_PERIOD,
     'usage': {
-        'cpu': [{
+        'instance': [{
             'desc': COMPUTE_METADATA,
             'vol': {
                 'qty': decimal.Decimal(1.0),
                 'unit': 'instance'}}]
-    }}]
+    },
+}]
 
 RATED_DATA = copy.deepcopy(COLLECTED_DATA)
-RATED_DATA[0]['usage']['cpu'][0]['rating'] = {
+RATED_DATA[0]['usage']['instance'][0]['rating'] = {
     'price': decimal.Decimal('0.42')}
 RATED_DATA[0]['usage']['image.size'][0]['rating'] = {
     'price': decimal.Decimal('0.1337')}
-RATED_DATA[1]['usage']['cpu'][0]['rating'] = {
+RATED_DATA[1]['usage']['instance'][0]['rating'] = {
     'price': decimal.Decimal('0.42')}
 
 
+DEFAULT_METRICS_CONF = {
+    "metrics": {
+        "cpu": {
+            "unit": "instance",
+            "alt_name": "instance",
+            "groupby": [
+                "id",
+                "project_id"
+            ],
+            "metadata": [
+                "flavor",
+                "flavor_id",
+                "vcpus"
+            ],
+            "mutate": "NUMBOOL",
+            "extra_args": {
+                "aggregation_method": "max",
+                "resource_type": "instance"
+            }
+        },
+        "image.size": {
+            "unit": "MiB",
+            "factor": "1/1048576",
+            "groupby": [
+                "id",
+                "project_id"
+            ],
+            "metadata": [
+                "container_format",
+                "disk_format"
+            ],
+            "extra_args": {
+                "aggregation_method": "max",
+                "resource_type": "image"
+            }
+        },
+        "volume.size": {
+            "unit": "GiB",
+            "groupby": [
+                "id",
+                "project_id"
+            ],
+            "metadata": [
+                "volume_type"
+            ],
+            "extra_args": {
+                "aggregation_method": "max",
+                "resource_type": "volume"
+            }
+        },
+        "network.outgoing.bytes": {
+            "unit": "MB",
+            "groupby": [
+                "id",
+                "project_id"
+            ],
+            "factor": "1/1000000",
+            "metadata": [
+                "instance_id"
+            ],
+            "extra_args": {
+                "aggregation_method": "max",
+                "resource_type": "instance_network_interface"
+            }
+        },
+        "network.incoming.bytes": {
+            "unit": "MB",
+            "groupby": [
+                "id",
+                "project_id"
+            ],
+            "factor": "1/1000000",
+            "metadata": [
+                "instance_id"
+            ],
+            "extra_args": {
+                "aggregation_method": "max",
+                "resource_type": "instance_network_interface"
+            }
+        },
+        "ip.floating": {
+            "unit": "ip",
+            "groupby": [
+                "id",
+                "project_id"
+            ],
+            "metadata": [
+                "state"
+            ],
+            "mutate": "NUMBOOL",
+            "extra_args": {
+                "aggregation_method": "max",
+                "resource_type": "network"
+            }
+        },
+        "radosgw.objects.size": {
+            "unit": "GiB",
+            "groupby": [
+                "id",
+                "project_id"
+            ],
+            "factor": "1/1073741824",
+            "extra_args": {
+                "aggregation_method": "max",
+                "resource_type": "ceph_account"
+            }
+        }
+    }
+}
+
+
 def split_storage_data(raw_data):
     final_data = []
     for frame in raw_data:
@@ -122,11 +235,11 @@ def split_storage_data(raw_data):
 # FIXME(sheeprine): storage is not using decimal for rates, we need to
 # transition to decimal.
 STORED_DATA = copy.deepcopy(COLLECTED_DATA)
-STORED_DATA[0]['usage']['cpu'][0]['rating'] = {
+STORED_DATA[0]['usage']['instance'][0]['rating'] = {
     'price': 0.42}
 STORED_DATA[0]['usage']['image.size'][0]['rating'] = {
     'price': 0.1337}
-STORED_DATA[1]['usage']['cpu'][0]['rating'] = {
+STORED_DATA[1]['usage']['instance'][0]['rating'] = {
     'price': 0.42}
 
 STORED_DATA = split_storage_data(STORED_DATA)
diff --git a/cloudkitty/tests/storage/test_hybrid_storage.py b/cloudkitty/tests/storage/test_hybrid_storage.py
index cafa2ea8..700072f8 100644
--- a/cloudkitty/tests/storage/test_hybrid_storage.py
+++ b/cloudkitty/tests/storage/test_hybrid_storage.py
@@ -21,18 +21,17 @@ import mock
 from gnocchiclient import exceptions as gexc
 
 from cloudkitty import storage
-from cloudkitty.storage.hybrid.backends import gnocchi as hgnocchi
 from cloudkitty import tests
-from cloudkitty.tests import samples
+from cloudkitty.tests import test_utils
 
 
 class BaseHybridStorageTest(tests.TestCase):
 
+    @mock.patch('cloudkitty.utils.load_conf', new=test_utils.load_conf)
     def setUp(self):
         super(BaseHybridStorageTest, self).setUp()
         self.conf.set_override('backend', 'hybrid', 'storage')
-        hgnocchi.METRICS_CONF = samples.METRICS_CONF
-        self.storage = storage.get_storage()
+        self.storage = storage.get_storage(conf=test_utils.load_conf())
         with mock.patch.object(
                 self.storage._hybrid_backend, 'init'):
             self.storage.init()
diff --git a/cloudkitty/tests/storage/test_storage.py b/cloudkitty/tests/storage/test_storage.py
index 1c2f11d0..0b0ddaaa 100644
--- a/cloudkitty/tests/storage/test_storage.py
+++ b/cloudkitty/tests/storage/test_storage.py
@@ -22,9 +22,9 @@ import sqlalchemy
 import testscenarios
 
 from cloudkitty import storage
-from cloudkitty.storage.hybrid.backends import gnocchi as hgnocchi
 from cloudkitty import tests
 from cloudkitty.tests import samples
+from cloudkitty.tests import test_utils
 from cloudkitty import utils as ck_utils
 
 
@@ -40,13 +40,13 @@ class StorageTest(tests.TestCase):
             cls.storage_scenarios)
 
     @mock.patch('cloudkitty.storage.hybrid.backends.gnocchi.gclient')
+    @mock.patch('cloudkitty.utils.load_conf', new=test_utils.load_conf)
     def setUp(self, gclient_mock):
         super(StorageTest, self).setUp()
-        hgnocchi.METRICS_CONF = samples.METRICS_CONF
         self._tenant_id = samples.TENANT
         self._other_tenant_id = '8d3ae50089ea4142-9c6e1269db6a0b64'
         self.conf.set_override('backend', self.storage_backend, 'storage')
-        self.storage = storage.get_storage()
+        self.storage = storage.get_storage(conf=test_utils.load_conf())
         self.storage.init()
 
     def insert_data(self):
@@ -234,10 +234,10 @@ class StorageTotalTest(StorageTest):
         total = self.storage.get_total(
             begin=begin,
             end=end,
-            service='cpu')
+            service='instance')
         self.assertEqual(1, len(total))
         self.assertEqual(0.84, total[0]["rate"])
-        self.assertEqual('cpu', total[0]["res_type"])
+        self.assertEqual('instance', total[0]["res_type"])
         self.assertEqual(begin, total[0]["begin"])
         self.assertEqual(end, total[0]["end"])
 
@@ -273,7 +273,7 @@ class StorageTotalTest(StorageTest):
         self.assertEqual(begin, total[0]["begin"])
         self.assertEqual(end, total[0]["end"])
         self.assertEqual(1.68, total[1]["rate"])
-        self.assertEqual('cpu', total[1]["res_type"])
+        self.assertEqual('instance', total[1]["res_type"])
         self.assertEqual(begin, total[1]["begin"])
         self.assertEqual(end, total[1]["end"])
 
@@ -298,12 +298,12 @@ class StorageTotalTest(StorageTest):
         self.assertEqual(end, total[1]["end"])
         self.assertEqual(0.84, total[2]["rate"])
         self.assertEqual(self._other_tenant_id, total[2]["tenant_id"])
-        self.assertEqual('cpu', total[2]["res_type"])
+        self.assertEqual('instance', total[2]["res_type"])
         self.assertEqual(begin, total[2]["begin"])
         self.assertEqual(end, total[2]["end"])
         self.assertEqual(0.84, total[3]["rate"])
         self.assertEqual(self._tenant_id, total[3]["tenant_id"])
-        self.assertEqual('cpu', total[3]["res_type"])
+        self.assertEqual('instance', total[3]["res_type"])
         self.assertEqual(begin, total[3]["begin"])
         self.assertEqual(end, total[3]["end"])
 
@@ -429,6 +429,9 @@ class StorageDataIntegrityTest(StorageTest):
         if 'image.size' in stored_data[0]['usage']:
             stored_data[0]['usage'], stored_data[1]['usage'] = (
                 stored_data[1]['usage'], stored_data[0]['usage'])
+        if 'image.size' in expected_data[0]['usage']:
+            expected_data[0]['usage'], expected_data[1]['usage'] = (
+                expected_data[1]['usage'], expected_data[0]['usage'])
         self.assertEqual(
             expected_data,
             stored_data)
diff --git a/cloudkitty/tests/test_orchestrator.py b/cloudkitty/tests/test_orchestrator.py
index 4dbc4243..d63f09ea 100644
--- a/cloudkitty/tests/test_orchestrator.py
+++ b/cloudkitty/tests/test_orchestrator.py
@@ -38,7 +38,6 @@ class OrchestratorTest(tests.TestCase):
         super(OrchestratorTest, self).setUp()
         messaging_conf = self.useFixture(conffixture.ConfFixture(self.conf))
         messaging_conf.transport_url = 'fake:/'
-        self.conf.set_override('backend', 'keystone', 'tenant_fetcher')
         self.conf.import_group('keystone_fetcher',
                                'cloudkitty.fetcher.keystone')
 
diff --git a/cloudkitty/tests/test_utils.py b/cloudkitty/tests/test_utils.py
index 9cb58b86..026d9dd1 100644
--- a/cloudkitty/tests/test_utils.py
+++ b/cloudkitty/tests/test_utils.py
@@ -24,6 +24,7 @@ import unittest
 import mock
 from oslo_utils import timeutils
 
+from cloudkitty.tests.samples import DEFAULT_METRICS_CONF
 from cloudkitty import utils as ck_utils
 
 
@@ -195,3 +196,7 @@ class ConvertUnitTest(unittest.TestCase):
     def test_convert_decimal(self):
         result = ck_utils.num2decimal(decimal.Decimal(2))
         self.assertEqual(result, decimal.Decimal(2))
+
+
+def load_conf(*args):
+    return DEFAULT_METRICS_CONF
diff --git a/cloudkitty/transformer/format.py b/cloudkitty/transformer/format.py
index a124f13a..381ac4f9 100644
--- a/cloudkitty/transformer/format.py
+++ b/cloudkitty/transformer/format.py
@@ -15,13 +15,23 @@
 #
 # @author: Stéphane Albert
 #
+
+from oslo_log import log
+
 from cloudkitty import transformer
 
 
+LOG = log.getLogger(__name__)
+
+
 class CloudKittyFormatTransformer(transformer.BaseTransformer):
-    def format_item(self, desc, unit, qty=1.0):
+    def format_item(self, groupby, metadata, unit, qty=1.0):
         data = {}
-        data['desc'] = desc
+        data['groupby'] = groupby
+        data['metadata'] = metadata
+        # For backward compatibility.
+        data['desc'] = data['groupby'].copy()
+        data['desc'].update(data['metadata'])
         data['vol'] = {'unit': unit, 'qty': qty}
 
         return data
diff --git a/cloudkitty/utils.py b/cloudkitty/utils.py
index eb16b943..a2073248 100644
--- a/cloudkitty/utils.py
+++ b/cloudkitty/utils.py
@@ -26,74 +26,24 @@ import contextlib
 import datetime
 import decimal
 import fractions
+import math
 import shutil
 import six
 import sys
 import tempfile
 import yaml
 
-from oslo_config import cfg
 from oslo_log import log as logging
-
 from oslo_utils import timeutils
 from six import moves
 from stevedore import extension
 
 
-COLLECTORS_NAMESPACE = 'cloudkitty.collector.backends'
-
 _ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f'
 _ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
 
 LOG = logging.getLogger(__name__)
 
-collect_opts = [
-    cfg.StrOpt('fetcher',
-               default='keystone',
-               deprecated_for_removal=True,
-               help='Project fetcher.'),
-    cfg.StrOpt('collector',
-               default='gnocchi',
-               deprecated_for_removal=True,
-               help='Data collector.'),
-    cfg.IntOpt('window',
-               default=1800,
-               deprecated_for_removal=True,
-               help='Number of samples to collect per call.'),
-    cfg.IntOpt('period',
-               default=3600,
-               deprecated_for_removal=True,
-               help='Rating period in seconds.'),
-    cfg.IntOpt('wait_periods',
-               default=2,
-               deprecated_for_removal=True,
-               help='Wait for N periods before collecting new data.'),
-    cfg.ListOpt('services',
-                default=[
-                    'compute',
-                    'volume',
-                    'network.bw.in',
-                    'network.bw.out',
-                    'network.floating',
-                    'image',
-                ],
-                deprecated_for_removal=True,
-                help='Services to monitor.'),
-    cfg.StrOpt('metrics_conf',
-               default='/etc/cloudkitty/metrics.yml',
-               help='Metrology configuration file.'),
-]
-
-storage_opts = [
-    cfg.StrOpt('backend',
-               default='sqlalchemy',
-               help='Name of the storage backend driver.')
-]
-
-CONF = cfg.CONF
-CONF.register_opts(collect_opts, 'collect')
-CONF.register_opts(storage_opts, 'storage')
-
 
 def isotime(at=None, subsecond=False):
     """Stringify time in ISO 8601 format."""
@@ -263,21 +213,22 @@ def refresh_stevedore(namespace=None):
         cache.clear()
 
 
-def check_time_state(timestamp=None, period=0, wait_time=0):
+def check_time_state(timestamp=None, period=0, wait_periods=0):
     if not timestamp:
         return get_month_start_timestamp()
 
     now = utcnow_ts()
     next_timestamp = timestamp + period
+    wait_time = wait_periods * period
     if next_timestamp + wait_time < now:
         return next_timestamp
     return 0
 
 
-def get_metrics_conf(conf_path):
-    """Return loaded yaml metrology configuration.
+def load_conf(conf_path):
+    """Return loaded yaml configuration.
 
-    In case not found metrics.yml file,
+    In case not found yaml file,
     return an empty dict.
     """
     # NOTE(mc): We can not raise any exception in this function as it called
@@ -286,10 +237,9 @@ def get_metrics_conf(conf_path):
     try:
         with open(conf_path) as conf:
             res = yaml.safe_load(conf)
-            res.update({'storage': CONF.storage.backend})
         return res or {}
     except Exception:
-        LOG.warning('Error when trying to retrieve yaml metrology conf file.')
+        LOG.warning("Error when trying to retrieve {} file.".format(conf_path))
         return {}
 
 
@@ -306,6 +256,21 @@ def tempdir(**kwargs):
                       six.text_type(e))
 
 
+def mutate(value, mode='NONE'):
+    """Mutate value according provided mode."""
+
+    if mode == 'NUMBOOL':
+        return float(value != 0.0)
+
+    if mode == 'FLOOR':
+        return math.floor(value)
+
+    if mode == 'CEIL':
+        return math.ceil(value)
+
+    return value
+
+
 def num2decimal(num):
     """Converts a number into a decimal.Decimal.
 
@@ -322,7 +287,7 @@ def num2decimal(num):
     return decimal.Decimal(num)
 
 
-def convert_unit(value, factor=1, offset=0):
+def convert_unit(value, factor, offset):
     """Return converted value depending on the provided factor and offset."""
     return num2decimal(value) * num2decimal(factor) + num2decimal(offset)
 
diff --git a/etc/cloudkitty/metrics.yml b/etc/cloudkitty/metrics.yml
index 790e4df8..ccef6e42 100644
--- a/etc/cloudkitty/metrics.yml
+++ b/etc/cloudkitty/metrics.yml
@@ -1,102 +1,85 @@
-name: OpenStack
-
-fetcher: keystone
-collector: gnocchi
-
-period: 3600
-wait_periods: 2
-window: 1800
-
-services_objects:
-  compute: instance
-  volume: volume
-  network.bw.out: instance_network_interface
-  network.bw.in: instance_network_interface
-  network.floating: network
-  image: image
-  radosgw.usage: ceph_account
-
 metrics:
-  vcpus:
-    resource: instance
-    unit: instance
-    factor: 1
-    aggregation_method: max
-    countable_unit: true
-
-  memory:
-    resource: instance
-    unit: instance
-    factor: 1
-    aggregation_method: max
-    countable_unit: true
-
   cpu:
-    resource: instance
     unit: instance
-    factor: 1
-    aggregation_method: max
-    countable_unit: true
-
-  disk.root.size:
-    resource: instance
-    unit: instance
-    factor: 1
-    aggregation_method: max
-    countable_unit: true
-
-  disk.ephemeral.size:
-    resource: instance
-    unit: instance
-    factor: 1
-    aggregation_method: max
-    countable_unit: true
+    alt_name: instance
+    groupby:
+      - id
+      - project_id
+    metadata:
+      - flavor
+      - flavor_id
+      - vcpus
+    mutate: NUMBOOL
+    extra_args:
+      aggregation_method: max
+      resource_type: instance
 
   image.size:
-    resource: image
     unit: MiB
     factor: 1/1048576
-    aggregation_method: max
-
-  image.download:
-    resource: image
-    unit: MiB
-    factor: 1/1048576
-    aggregation_method: max
-
-  image.serve:
-    resource: image
-    unit: MiB
-    factor: 1/1048576
-    aggregation_method: max
+    groupby:
+      - id
+      - project_id
+    metadata:
+      - container_format
+      - disk_format
+    extra_args:
+      aggregation_method: max
+      resource_type: image
 
   volume.size:
-    resource: volume
     unit: GiB
-    factor: 1
-    aggregation_method: max
+    groupby:
+      - id
+      - project_id
+    metadata:
+      - volume_type
+    extra_args:
+      aggregation_method: max
+      resource_type: volume
 
   network.outgoing.bytes:
-    resource: instance_network_interface
     unit: MB
+    groupby:
+      - id
+      - project_id
     factor: 1/1000000
-    aggregation_method: max
+    metadata:
+      - instance_id
+    extra_args:
+      aggregation_method: max
+      resource_type: instance_network_interface
 
   network.incoming.bytes:
-    resource: instance_network_interface
     unit: MB
+    groupby:
+      - id
+      - project_id
     factor: 1/1000000
-    aggregation_method: max
+    metadata:
+      - instance_id
+    extra_args:
+      aggregation_method: max
+      resource_type: instance_network_interface
 
   ip.floating:
-    resource: network
     unit: ip
-    factor: 1
-    aggregation_method: max
-    countable_unit: true
+    groupby:
+      - id
+      - project_id
+    metadata:
+      - state
+    mutate: NUMBOOL
+    extra_args:
+      aggregation_method: max
+      resource_type: network
 
   radosgw.objects.size:
-    resource: ceph_account
     unit: GiB
+    groupby:
+      - id
+      - project_id
     factor: 1/1073741824
-    aggregation_method: max
+    extra_args:
+      aggregation_method: max
+      resource_type: ceph_account
diff --git a/releasenotes/notes/improve-metrics-configuration-271102366f8e6fe7.yaml b/releasenotes/notes/improve-metrics-configuration-271102366f8e6fe7.yaml
new file mode 100644
index 00000000..9ef0995c
--- /dev/null
+++ b/releasenotes/notes/improve-metrics-configuration-271102366f8e6fe7.yaml
@@ -0,0 +1,4 @@
+features:
+  - |
+    The format of the 'metrics.yml' configuration file has been improved,
+    and will be stable.
diff --git a/requirements.txt b/requirements.txt
index e6f70740..059174d2 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -27,3 +27,4 @@ SQLAlchemy<1.1.0,>=1.0.10 # MIT
 six>=1.9.0 # MIT
 stevedore>=1.5.0 # Apache-2.0
 tooz>=1.28.0 # Apache-2.0
+voluptuous>=0.11.1,<1.0.0 # BSD-3
diff --git a/setup.cfg b/setup.cfg
index f184016f..cbe34c62 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -50,7 +50,7 @@ cloudkitty.collector.backends =
     monasca = cloudkitty.collector.monasca:MonascaCollector
     meta = cloudkitty.collector.meta:MetaCollector
 
-cloudkitty.tenant.fetchers =
+cloudkitty.fetchers =
     fake = cloudkitty.fetcher.fake:FakeFetcher
     keystone = cloudkitty.fetcher.keystone:KeystoneFetcher
     source = cloudkitty.fetcher.source:SourceFetcher