Remove transformers from the codebase

Since data frames are now handled as objects, transformers are no longer
required. This simplifies the global codebase.

Story: 2005890
Task: 36075
Change-Id: I76d9117bd95d80e51ca95804c999f145e65c3a2d

cloudkitty/collector/__init__.py

@@ -31,7 +31,6 @@ from voluptuous import Optional
 from voluptuous import Required
 from voluptuous import Schema
 
-from cloudkitty import transformer
 from cloudkitty import utils as ck_utils
 
 
@@ -101,15 +100,12 @@ METRIC_BASE_SCHEMA = {
 }
 
 
-def get_collector(transformers=None):
+def get_collector():
     metrics_conf = ck_utils.load_conf(CONF.collect.metrics_conf)
-    if not transformers:
-        transformers = transformer.get_transformers()
     collector_args = {
         'period': CONF.collect.period,
-        'transformers': transformers,
+        'conf': metrics_conf,
     }
-    collector_args.update({'conf': metrics_conf})
     return driver.DriverManager(
         COLLECTORS_NAMESPACE,
         CONF.collect.collector,
@@ -132,7 +128,6 @@ def get_metrics_based_collector_metadata():
     Results are based on enabled collector and metrics in CONF.
     """
     metrics_conf = ck_utils.load_conf(CONF.collect.metrics_conf)
-    transformers = transformer.get_transformers()
     collector = get_collector_without_invoke()
     metadata = {}
     if 'metrics' in metrics_conf:
@@ -140,23 +135,11 @@ def get_metrics_based_collector_metadata():
             alt_name = metric.get('alt_name', metric_name)
             metadata[alt_name] = collector.get_metadata(
                 metric_name,
-                transformers,
                 metrics_conf,
             )
     return metadata
 
 
-class TransformerDependencyError(Exception):
-    """Raised when a collector can't find a mandatory transformer."""
-
-    def __init__(self, collector, transformer):
-        super(TransformerDependencyError, self).__init__(
-            "Transformer '%s' not found, but required by %s" % (transformer,
-                                                                collector))
-        self.collector = collector
-        self.transformer = transformer
-
-
 class NoDataCollected(Exception):
     """Raised when the collection returned no data.
 
@@ -173,11 +156,9 @@ class NoDataCollected(Exception):
 @six.add_metaclass(abc.ABCMeta)
 class BaseCollector(object):
     collector_name = None
-    dependencies = ['CloudKittyFormatTransformer']
 
-    def __init__(self, transformers, **kwargs):
+    def __init__(self, **kwargs):
         try:
-            self.transformers = transformers
             self.period = kwargs['period']
             self.conf = self.check_configuration(kwargs['conf'])
         except KeyError as e:
@@ -188,18 +169,6 @@ class BaseCollector(object):
             LOG.error('Problem while checking configurations.', v)
             raise v
 
-        self._check_transformers()
-        self.t_cloudkitty = self.transformers['CloudKittyFormatTransformer']
-
-    def _check_transformers(self):
-        """Check for transformer prerequisites
-
-        """
-        for dependency in self.dependencies:
-            if dependency not in self.transformers:
-                raise TransformerDependencyError(self.collector_name,
-                                                 dependency)
-
     @staticmethod
     def check_configuration(conf):
         """Checks and validates metric configuration.
@@ -229,7 +198,7 @@ class BaseCollector(object):
         return trans_resource
 
     @classmethod
-    def get_metadata(cls, resource_name, transformers):
+    def get_metadata(cls, resource_name):
         """Return metadata about collected resource as a dict.
 
            Dict object should contain:
@@ -247,8 +216,7 @@ class BaseCollector(object):
         provided in the metric conf at initialization.
         (Available in ``self.conf['groupby']`` and ``self.conf['metadata']``).
 
-        Returns a list of items formatted with
+        Returns a list of cloudkitty.dataframe.DataPoint objects.
-        ``CloudKittyFormatTransformer.format_item``.
 
         :param metric_name: Name of the metric to fetch
         :type metric_name: str

cloudkitty/collector/gnocchi.py

@@ -29,6 +29,7 @@ from voluptuous import Required
 from voluptuous import Schema
 
 from cloudkitty import collector
+from cloudkitty import dataframe
 from cloudkitty import utils as ck_utils
 
 
@@ -116,8 +117,8 @@ class GnocchiCollector(collector.BaseCollector):
 
     collector_name = 'gnocchi'
 
-    def __init__(self, transformers, **kwargs):
+    def __init__(self, **kwargs):
-        super(GnocchiCollector, self).__init__(transformers, **kwargs)
+        super(GnocchiCollector, self).__init__(**kwargs)
 
         adapter_options = {'connect_retries': 3}
         if CONF.collector_gnocchi.gnocchi_auth_type == 'keystone':
@@ -164,9 +165,8 @@ class GnocchiCollector(collector.BaseCollector):
         return output
 
     @classmethod
-    def get_metadata(cls, resource_name, transformers, conf):
+    def get_metadata(cls, resource_name, conf):
-        info = super(GnocchiCollector, cls).get_metadata(resource_name,
+        info = super(GnocchiCollector, cls).get_metadata(resource_name)
-                                                         transformers)
         try:
             info["metadata"].extend(
                 conf[resource_name]['groupby']
@@ -392,11 +392,11 @@ class GnocchiCollector(collector.BaseCollector):
                             project_id, start, end, e),
                     )
                     continue
-                data = self.t_cloudkitty.format_item(
+                formated_resources.append(dataframe.DataPoint(
+                    met['unit'],
+                    qty,
+                    0,
                     groupby,
                     metadata,
-                    met['unit'],
+                ))
-                    qty=qty,
-                )
-                formated_resources.append(data)
         return formated_resources

cloudkitty/collector/monasca.py

@@ -25,7 +25,7 @@ from voluptuous import Required
 from voluptuous import Schema
 
 from cloudkitty import collector
-from cloudkitty import transformer
+from cloudkitty import dataframe
 from cloudkitty import utils as ck_utils
 
 
@@ -94,8 +94,8 @@ class MonascaCollector(collector.BaseCollector):
 
         return output
 
-    def __init__(self, transformers, **kwargs):
+    def __init__(self, **kwargs):
-        super(MonascaCollector, self).__init__(transformers, **kwargs)
+        super(MonascaCollector, self).__init__(**kwargs)
 
         self.auth = ks_loading.load_auth_from_conf_options(
             CONF,
@@ -129,7 +129,7 @@ class MonascaCollector(collector.BaseCollector):
                 return endpoint.url
         return None
 
-    def _get_metadata(self, metric_name, transformers, conf):
+    def _get_metadata(self, metric_name, conf):
         info = {}
         info['unit'] = conf['metrics'][metric_name]['unit']
 
@@ -141,12 +141,9 @@ class MonascaCollector(collector.BaseCollector):
     # NOTE(lukapeschke) if anyone sees a better way to do this,
     # please make a patch
     @classmethod
-    def get_metadata(cls, resource_type, transformers, conf):
+    def get_metadata(cls, resource_type, conf):
-        args = {
+        tmp = cls(period=conf['period'])
-            'transformers': transformer.get_transformers(),
+        return tmp._get_metadata(resource_type, conf)
-            'period': conf['period']}
-        tmp = cls(**args)
-        return tmp._get_metadata(resource_type, transformers, conf)
 
     def _get_dimensions(self, metric_name, project_id, q_filter):
         dimensions = {}
@@ -267,11 +264,11 @@ class MonascaCollector(collector.BaseCollector):
             if len(d['statistics']):
                 metadata, groupby, qty = self._format_data(
                     met, d, resources_info)
-                data = self.t_cloudkitty.format_item(
+                formated_resources.append(dataframe.DataPoint(
+                    met['unit'],
+                    qty,
+                    0,
                     groupby,
                     metadata,
-                    met['unit'],
+                ))
-                    qty=qty,
-                )
-                formated_resources.append(data)
         return formated_resources

cloudkitty/collector/prometheus.py

@@ -26,6 +26,7 @@ from cloudkitty import collector
 from cloudkitty.collector.exceptions import CollectError
 from cloudkitty.common.prometheus_client import PrometheusClient
 from cloudkitty.common.prometheus_client import PrometheusResponseError
+from cloudkitty import dataframe
 from cloudkitty import utils as ck_utils
 
 
@@ -76,8 +77,8 @@ PROMETHEUS_EXTRA_SCHEMA = {
 class PrometheusCollector(collector.BaseCollector):
     collector_name = 'prometheus'
 
-    def __init__(self, transformers, **kwargs):
+    def __init__(self, **kwargs):
-        super(PrometheusCollector, self).__init__(transformers, **kwargs)
+        super(PrometheusCollector, self).__init__(**kwargs)
         url = CONF.collector_prometheus.prometheus_url
 
         user = CONF.collector_prometheus.prometheus_user
@@ -176,13 +177,12 @@ class PrometheusCollector(collector.BaseCollector):
                 item,
             )
 
-            item = self.t_cloudkitty.format_item(
+            formatted_resources.append(dataframe.DataPoint(
+                self.conf[metric_name]['unit'],
+                qty,
+                0,
                 groupby,
                 metadata,
-                self.conf[metric_name]['unit'],
+            ))
-                qty=qty,
-            )
-
-            formatted_resources.append(item)
 
         return formatted_resources

cloudkitty/orchestrator.py

@@ -39,7 +39,6 @@ from cloudkitty import extension_manager
 from cloudkitty import messaging
 from cloudkitty import storage
 from cloudkitty import storage_state as state
-from cloudkitty import transformer
 from cloudkitty import tzutils
 from cloudkitty import utils as ck_utils
 
@@ -334,8 +333,7 @@ class Orchestrator(cotyledon.Service):
             invoke_on_load=True,
         ).driver
 
-        transformers = transformer.get_transformers()
+        self.collector = collector.get_collector()
-        self.collector = collector.get_collector(transformers)
         self.storage = storage.get_storage()
         self._state = state.StateManager()
 

cloudkitty/tests/collectors/test_gnocchi.py

@@ -17,7 +17,6 @@
 from cloudkitty.collector import gnocchi
 from cloudkitty import tests
 from cloudkitty.tests import samples
-from cloudkitty import transformer
 
 
 class GnocchiCollectorTest(tests.TestCase):
@@ -29,7 +28,6 @@ class GnocchiCollectorTest(tests.TestCase):
             'gnocchi_auth_type', 'basic', 'collector_gnocchi')
 
         self.collector = gnocchi.GnocchiCollector(
-            transformer.get_transformers(),
             period=3600,
             conf=samples.DEFAULT_METRICS_CONF,
         )

cloudkitty/tests/collectors/test_monasca.py

@@ -18,7 +18,6 @@ import mock
 
 from cloudkitty.collector import monasca as mon_collector
 from cloudkitty import tests
-from cloudkitty import transformer
 
 
 class MonascaCollectorTest(tests.TestCase):
@@ -50,7 +49,6 @@ class MonascaCollectorTest(tests.TestCase):
                 'MonascaCollector._get_monasca_endpoint',
                 return_value='http://noop'):
             self.collector = mon_collector.MonascaCollector(
-                transformer.get_transformers(),
                 period=3600,
                 conf=conf,
             )

cloudkitty/tests/collectors/test_prometheus.py

@@ -24,7 +24,6 @@ from cloudkitty.common.prometheus_client import PrometheusResponseError
 from cloudkitty import dataframe
 from cloudkitty import tests
 from cloudkitty.tests import samples
-from cloudkitty import transformer
 
 
 class PrometheusCollectorTest(tests.TestCase):
@@ -53,8 +52,7 @@ class PrometheusCollectorTest(tests.TestCase):
                 }
             }
         }
-        transformers = transformer.get_transformers()
+        self.collector = prometheus.PrometheusCollector(**args)
-        self.collector = prometheus.PrometheusCollector(transformers, **args)
 
     def test_fetch_all_build_query(self):
         query = (

cloudkitty/tests/transformers/__init__.py

@@ -1,54 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2016 Objectif Libre
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-#
-from cloudkitty.tests import samples
-from cloudkitty import transformer
-
-
-class Transformer(transformer.BaseTransformer):
-    compute_map = {
-        'name': ['name', 'display_name'],
-        'flavor': ['flavor', 'flavor.name', 'instance_type'],
-        'vcpus': ['vcpus'],
-        'memory': ['memory', 'memory_mb'],
-        'image_id': ['image_id', 'image.id', 'image_meta.base_image_ref'],
-        'availability_zone': [
-            'availability_zone',
-            'OS-EXT-AZ.availability_zone'],
-    }
-    volume_map = {
-        'volume_id': ['volume_id'],
-        'name': ['display_name'],
-        'availability_zone': ['availability_zone'],
-        'size': ['size'],
-    }
-    test_map = {'test': lambda x, y: 'ok'}
-
-    def _strip_network(self, res_metadata):
-        return {'test': 'ok'}
-
-
-class TransformerMeta(Transformer):
-    metadata_item = 'metadata'
-
-
-class EmptyClass(object):
-    pass
-
-
-class ClassWithAttr(object):
-    def __init__(self, items=samples.COMPUTE_METADATA):
-        for key, val in items.items():
-            setattr(self, key, val)

cloudkitty/tests/transformers/test_base.py

@@ -1,68 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2016 Objectif Libre
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-#
-import copy
-
-from cloudkitty import tests
-from cloudkitty.tests import samples
-from cloudkitty.tests import transformers as t_transformers
-
-TRANS_METADATA = {
-    'availability_zone': 'nova',
-    'flavor': 'm1.nano',
-    'image_id': 'f5600101-8fa2-4864-899e-ebcb7ed6b568',
-    'memory': '64',
-    'name': 'prod1',
-    'vcpus': '1'}
-
-
-class TransformerBaseTest(tests.TestCase):
-    def test_strip_resource_on_dict(self):
-        metadata = copy.deepcopy(samples.COMPUTE_METADATA)
-        t_test = t_transformers.Transformer()
-        result = t_test.strip_resource_data('compute', metadata)
-        self.assertEqual(TRANS_METADATA, result)
-
-    def test_strip_resource_with_no_rules(self):
-        metadata = copy.deepcopy(samples.COMPUTE_METADATA)
-        t_test = t_transformers.Transformer()
-        result = t_test.strip_resource_data('unknown', metadata)
-        self.assertEqual(samples.COMPUTE_METADATA, result)
-
-    def test_strip_resource_with_func(self):
-        metadata = {'test': 'dummy'}
-        t_test = t_transformers.Transformer()
-        result = t_test.strip_resource_data('test', metadata)
-        self.assertEqual({'test': 'ok'}, result)
-
-    def test_strip_resource_with_stripping_function(self):
-        metadata = {}
-        t_test = t_transformers.Transformer()
-        result = t_test.strip_resource_data('network', metadata)
-        self.assertEqual({'test': 'ok'}, result)
-
-    def test_strip_resource_with_subitem(self):
-        test_obj = t_transformers.EmptyClass()
-        test_obj.metadata = copy.deepcopy(samples.COMPUTE_METADATA)
-        t_test = t_transformers.TransformerMeta()
-        result = t_test.strip_resource_data('compute', test_obj)
-        self.assertEqual(TRANS_METADATA, result)
-
-    def test_strip_resource_with_attributes(self):
-        test_obj = t_transformers.EmptyClass()
-        test_obj.metadata = t_transformers.ClassWithAttr()
-        t_test = t_transformers.TransformerMeta()
-        result = t_test.strip_resource_data('compute', test_obj)
-        self.assertEqual(TRANS_METADATA, result)

cloudkitty/transformer/__init__.py

@@ -1,69 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2014 Objectif Libre
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-#
-import abc
-
-import six
-from stevedore import extension
-
-TRANSFORMERS_NAMESPACE = 'cloudkitty.transformers'
-
-
-def get_transformers():
-    transformers = {}
-    transformer_exts = extension.ExtensionManager(
-        TRANSFORMERS_NAMESPACE,
-        invoke_on_load=True)
-    for transformer in transformer_exts:
-        t_name = transformer.name
-        t_obj = transformer.obj
-        transformers[t_name] = t_obj
-    return transformers
-
-
-@six.add_metaclass(abc.ABCMeta)
-class BaseTransformer(object):
-    metadata_item = ''
-
-    def generic_strip(self, datatype, data):
-        metadata = getattr(data, self.metadata_item, data)
-        mappings = getattr(self, datatype + '_map', {})
-        result = {}
-        for key, transform in mappings.items():
-            if isinstance(transform, list):
-                for meta_key in transform:
-                    if key not in result or result[key] is None:
-                        try:
-                            data = getattr(metadata, meta_key)
-                        except AttributeError:
-                            data = metadata.get(meta_key)
-                        result[key] = data
-            else:
-                trans_data = transform(self, metadata)
-                if trans_data:
-                    result[key] = trans_data
-        return result
-
-    def strip_resource_data(self, res_type, res_data):
-        res_type = res_type.replace('.', '_')
-        strip_func = getattr(self, '_strip_' + res_type, None)
-        if strip_func:
-            return strip_func(res_data)
-        return self.generic_strip(res_type, res_data) or res_data
-
-    def get_metadata(self, res_type):
-        """Return list of metadata available for given resource type."""
-
-        return []

cloudkitty/transformer/format.py

@@ -1,43 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2014 Objectif Libre
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-#
-
-from oslo_log import log
-
-from cloudkitty import dataframe
-from cloudkitty import transformer
-
-
-LOG = log.getLogger(__name__)
-
-
-class CloudKittyFormatTransformer(transformer.BaseTransformer):
-    def format_item(self, groupby, metadata, unit, qty=1.0):
-        # data = {}
-        # data['groupby'] = groupby
-        # data['metadata'] = metadata
-        # # For backward compatibility.
-        # data['desc'] = data['groupby'].copy()
-        # data['desc'].update(data['metadata'])
-        # data['vol'] = {'unit': unit, 'qty': qty}
-
-        return dataframe.DataPoint(unit, qty, 0, groupby, metadata)
-        # return data
-
-    def format_service(self, service, items):
-        data = {}
-        data[service] = items
-
-        return data

doc/source/developer/collector.rst

@@ -61,8 +61,8 @@ following prototype:
 .. autoclass:: cloudkitty.collector.BaseCollector
    :members: fetch_all
 
-This method is supposed to return a list of objects formatted by
+This method is supposed to return a list of
-``CloudKittyFormatTransformer``.
+``cloudkitty.dataframe.DataPoint`` objects.
 
 Example code of a basic collector:
 
@@ -79,11 +79,12 @@ Example code of a basic collector:
             data = []
             for CONDITION:
                 # do stuff
-                data.append(self.t_cloudkitty.format_item(
+                data.append(dataframe.DataPoint(
+                    unit,
+                    qty, # int, float, decimal.Decimal or str
+                    0, # price
                     groupby, # dict
                     metadata, # dict
-                    unit, # str
-                    qty=qty, # int / float
                 ))
 
             return data

releasenotes/notes/remove-transformers-8d9949ed3088b055.yaml

@@ -0,0 +1,5 @@
+---
+other:
+  - |
+    Since data frames are now represented as objects internally, transformers
+    are not used anymore and have been completely removed from the codebase.

setup.cfg

@@ -57,9 +57,6 @@ cloudkitty.fetchers =
     gnocchi = cloudkitty.fetcher.gnocchi:GnocchiFetcher
     prometheus = cloudkitty.fetcher.prometheus:PrometheusFetcher
 
-cloudkitty.transformers =
-    CloudKittyFormatTransformer = cloudkitty.transformer.format:CloudKittyFormatTransformer
-
 cloudkitty.rating.processors =
     noop = cloudkitty.rating.noop:Noop
     hashmap = cloudkitty.rating.hash:HashMap