From 492ec063a72f3d9781c042bf5efccd68b262b918 Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Fri, 2 Aug 2019 16:19:00 +0200 Subject: [PATCH] Remove transformers from the codebase Since data frames are now handled as objects, transformers are no longer required. This simplifies the global codebase. Story: 2005890 Task: 36075 Change-Id: I76d9117bd95d80e51ca95804c999f145e65c3a2d --- cloudkitty/collector/__init__.py | 42 ++--------- cloudkitty/collector/gnocchi.py | 20 +++--- cloudkitty/collector/monasca.py | 27 ++++---- cloudkitty/collector/prometheus.py | 16 ++--- cloudkitty/orchestrator.py | 4 +- cloudkitty/tests/collectors/test_gnocchi.py | 2 - cloudkitty/tests/collectors/test_monasca.py | 2 - .../tests/collectors/test_prometheus.py | 4 +- cloudkitty/tests/transformers/__init__.py | 54 --------------- cloudkitty/tests/transformers/test_base.py | 68 ------------------ cloudkitty/transformer/__init__.py | 69 ------------------- cloudkitty/transformer/format.py | 43 ------------ doc/source/developer/collector.rst | 11 +-- .../remove-transformers-8d9949ed3088b055.yaml | 5 ++ setup.cfg | 3 - 15 files changed, 48 insertions(+), 322 deletions(-) delete mode 100644 cloudkitty/tests/transformers/__init__.py delete mode 100644 cloudkitty/tests/transformers/test_base.py delete mode 100644 cloudkitty/transformer/__init__.py delete mode 100644 cloudkitty/transformer/format.py create mode 100644 releasenotes/notes/remove-transformers-8d9949ed3088b055.yaml diff --git a/cloudkitty/collector/__init__.py b/cloudkitty/collector/__init__.py index 04d4fb18..595def33 100644 --- a/cloudkitty/collector/__init__.py +++ b/cloudkitty/collector/__init__.py @@ -31,7 +31,6 @@ from voluptuous import Optional from voluptuous import Required from voluptuous import Schema -from cloudkitty import transformer from cloudkitty import utils as ck_utils @@ -101,15 +100,12 @@ METRIC_BASE_SCHEMA = { } -def get_collector(transformers=None): +def get_collector(): metrics_conf = ck_utils.load_conf(CONF.collect.metrics_conf) - if not transformers: - transformers = transformer.get_transformers() collector_args = { 'period': CONF.collect.period, - 'transformers': transformers, + 'conf': metrics_conf, } - collector_args.update({'conf': metrics_conf}) return driver.DriverManager( COLLECTORS_NAMESPACE, CONF.collect.collector, @@ -132,7 +128,6 @@ def get_metrics_based_collector_metadata(): Results are based on enabled collector and metrics in CONF. """ metrics_conf = ck_utils.load_conf(CONF.collect.metrics_conf) - transformers = transformer.get_transformers() collector = get_collector_without_invoke() metadata = {} if 'metrics' in metrics_conf: @@ -140,23 +135,11 @@ def get_metrics_based_collector_metadata(): alt_name = metric.get('alt_name', metric_name) metadata[alt_name] = collector.get_metadata( metric_name, - transformers, metrics_conf, ) return metadata -class TransformerDependencyError(Exception): - """Raised when a collector can't find a mandatory transformer.""" - - def __init__(self, collector, transformer): - super(TransformerDependencyError, self).__init__( - "Transformer '%s' not found, but required by %s" % (transformer, - collector)) - self.collector = collector - self.transformer = transformer - - class NoDataCollected(Exception): """Raised when the collection returned no data. @@ -173,11 +156,9 @@ class NoDataCollected(Exception): @six.add_metaclass(abc.ABCMeta) class BaseCollector(object): collector_name = None - dependencies = ['CloudKittyFormatTransformer'] - def __init__(self, transformers, **kwargs): + def __init__(self, **kwargs): try: - self.transformers = transformers self.period = kwargs['period'] self.conf = self.check_configuration(kwargs['conf']) except KeyError as e: @@ -188,18 +169,6 @@ class BaseCollector(object): LOG.error('Problem while checking configurations.', v) raise v - self._check_transformers() - self.t_cloudkitty = self.transformers['CloudKittyFormatTransformer'] - - def _check_transformers(self): - """Check for transformer prerequisites - - """ - for dependency in self.dependencies: - if dependency not in self.transformers: - raise TransformerDependencyError(self.collector_name, - dependency) - @staticmethod def check_configuration(conf): """Checks and validates metric configuration. @@ -229,7 +198,7 @@ class BaseCollector(object): return trans_resource @classmethod - def get_metadata(cls, resource_name, transformers): + def get_metadata(cls, resource_name): """Return metadata about collected resource as a dict. Dict object should contain: @@ -247,8 +216,7 @@ class BaseCollector(object): provided in the metric conf at initialization. (Available in ``self.conf['groupby']`` and ``self.conf['metadata']``). - Returns a list of items formatted with - ``CloudKittyFormatTransformer.format_item``. + Returns a list of cloudkitty.dataframe.DataPoint objects. :param metric_name: Name of the metric to fetch :type metric_name: str diff --git a/cloudkitty/collector/gnocchi.py b/cloudkitty/collector/gnocchi.py index d47ee2fb..5fe05084 100644 --- a/cloudkitty/collector/gnocchi.py +++ b/cloudkitty/collector/gnocchi.py @@ -29,6 +29,7 @@ from voluptuous import Required from voluptuous import Schema from cloudkitty import collector +from cloudkitty import dataframe from cloudkitty import utils as ck_utils @@ -116,8 +117,8 @@ class GnocchiCollector(collector.BaseCollector): collector_name = 'gnocchi' - def __init__(self, transformers, **kwargs): - super(GnocchiCollector, self).__init__(transformers, **kwargs) + def __init__(self, **kwargs): + super(GnocchiCollector, self).__init__(**kwargs) adapter_options = {'connect_retries': 3} if CONF.collector_gnocchi.gnocchi_auth_type == 'keystone': @@ -164,9 +165,8 @@ class GnocchiCollector(collector.BaseCollector): return output @classmethod - def get_metadata(cls, resource_name, transformers, conf): - info = super(GnocchiCollector, cls).get_metadata(resource_name, - transformers) + def get_metadata(cls, resource_name, conf): + info = super(GnocchiCollector, cls).get_metadata(resource_name) try: info["metadata"].extend( conf[resource_name]['groupby'] @@ -392,11 +392,11 @@ class GnocchiCollector(collector.BaseCollector): project_id, start, end, e), ) continue - data = self.t_cloudkitty.format_item( + formated_resources.append(dataframe.DataPoint( + met['unit'], + qty, + 0, groupby, metadata, - met['unit'], - qty=qty, - ) - formated_resources.append(data) + )) return formated_resources diff --git a/cloudkitty/collector/monasca.py b/cloudkitty/collector/monasca.py index b2141c6d..d58dab78 100644 --- a/cloudkitty/collector/monasca.py +++ b/cloudkitty/collector/monasca.py @@ -25,7 +25,7 @@ from voluptuous import Required from voluptuous import Schema from cloudkitty import collector -from cloudkitty import transformer +from cloudkitty import dataframe from cloudkitty import utils as ck_utils @@ -94,8 +94,8 @@ class MonascaCollector(collector.BaseCollector): return output - def __init__(self, transformers, **kwargs): - super(MonascaCollector, self).__init__(transformers, **kwargs) + def __init__(self, **kwargs): + super(MonascaCollector, self).__init__(**kwargs) self.auth = ks_loading.load_auth_from_conf_options( CONF, @@ -129,7 +129,7 @@ class MonascaCollector(collector.BaseCollector): return endpoint.url return None - def _get_metadata(self, metric_name, transformers, conf): + def _get_metadata(self, metric_name, conf): info = {} info['unit'] = conf['metrics'][metric_name]['unit'] @@ -141,12 +141,9 @@ class MonascaCollector(collector.BaseCollector): # NOTE(lukapeschke) if anyone sees a better way to do this, # please make a patch @classmethod - def get_metadata(cls, resource_type, transformers, conf): - args = { - 'transformers': transformer.get_transformers(), - 'period': conf['period']} - tmp = cls(**args) - return tmp._get_metadata(resource_type, transformers, conf) + def get_metadata(cls, resource_type, conf): + tmp = cls(period=conf['period']) + return tmp._get_metadata(resource_type, conf) def _get_dimensions(self, metric_name, project_id, q_filter): dimensions = {} @@ -267,11 +264,11 @@ class MonascaCollector(collector.BaseCollector): if len(d['statistics']): metadata, groupby, qty = self._format_data( met, d, resources_info) - data = self.t_cloudkitty.format_item( + formated_resources.append(dataframe.DataPoint( + met['unit'], + qty, + 0, groupby, metadata, - met['unit'], - qty=qty, - ) - formated_resources.append(data) + )) return formated_resources diff --git a/cloudkitty/collector/prometheus.py b/cloudkitty/collector/prometheus.py index f2876f3a..5add5620 100644 --- a/cloudkitty/collector/prometheus.py +++ b/cloudkitty/collector/prometheus.py @@ -26,6 +26,7 @@ from cloudkitty import collector from cloudkitty.collector.exceptions import CollectError from cloudkitty.common.prometheus_client import PrometheusClient from cloudkitty.common.prometheus_client import PrometheusResponseError +from cloudkitty import dataframe from cloudkitty import utils as ck_utils @@ -76,8 +77,8 @@ PROMETHEUS_EXTRA_SCHEMA = { class PrometheusCollector(collector.BaseCollector): collector_name = 'prometheus' - def __init__(self, transformers, **kwargs): - super(PrometheusCollector, self).__init__(transformers, **kwargs) + def __init__(self, **kwargs): + super(PrometheusCollector, self).__init__(**kwargs) url = CONF.collector_prometheus.prometheus_url user = CONF.collector_prometheus.prometheus_user @@ -176,13 +177,12 @@ class PrometheusCollector(collector.BaseCollector): item, ) - item = self.t_cloudkitty.format_item( + formatted_resources.append(dataframe.DataPoint( + self.conf[metric_name]['unit'], + qty, + 0, groupby, metadata, - self.conf[metric_name]['unit'], - qty=qty, - ) - - formatted_resources.append(item) + )) return formatted_resources diff --git a/cloudkitty/orchestrator.py b/cloudkitty/orchestrator.py index df57f56a..f0abf2c3 100644 --- a/cloudkitty/orchestrator.py +++ b/cloudkitty/orchestrator.py @@ -39,7 +39,6 @@ from cloudkitty import extension_manager from cloudkitty import messaging from cloudkitty import storage from cloudkitty import storage_state as state -from cloudkitty import transformer from cloudkitty import tzutils from cloudkitty import utils as ck_utils @@ -334,8 +333,7 @@ class Orchestrator(cotyledon.Service): invoke_on_load=True, ).driver - transformers = transformer.get_transformers() - self.collector = collector.get_collector(transformers) + self.collector = collector.get_collector() self.storage = storage.get_storage() self._state = state.StateManager() diff --git a/cloudkitty/tests/collectors/test_gnocchi.py b/cloudkitty/tests/collectors/test_gnocchi.py index 9f6b5a0f..671f5ef1 100644 --- a/cloudkitty/tests/collectors/test_gnocchi.py +++ b/cloudkitty/tests/collectors/test_gnocchi.py @@ -17,7 +17,6 @@ from cloudkitty.collector import gnocchi from cloudkitty import tests from cloudkitty.tests import samples -from cloudkitty import transformer class GnocchiCollectorTest(tests.TestCase): @@ -29,7 +28,6 @@ class GnocchiCollectorTest(tests.TestCase): 'gnocchi_auth_type', 'basic', 'collector_gnocchi') self.collector = gnocchi.GnocchiCollector( - transformer.get_transformers(), period=3600, conf=samples.DEFAULT_METRICS_CONF, ) diff --git a/cloudkitty/tests/collectors/test_monasca.py b/cloudkitty/tests/collectors/test_monasca.py index 2dad9e63..dbb202c6 100644 --- a/cloudkitty/tests/collectors/test_monasca.py +++ b/cloudkitty/tests/collectors/test_monasca.py @@ -18,7 +18,6 @@ import mock from cloudkitty.collector import monasca as mon_collector from cloudkitty import tests -from cloudkitty import transformer class MonascaCollectorTest(tests.TestCase): @@ -50,7 +49,6 @@ class MonascaCollectorTest(tests.TestCase): 'MonascaCollector._get_monasca_endpoint', return_value='http://noop'): self.collector = mon_collector.MonascaCollector( - transformer.get_transformers(), period=3600, conf=conf, ) diff --git a/cloudkitty/tests/collectors/test_prometheus.py b/cloudkitty/tests/collectors/test_prometheus.py index 27e29a0c..07c0ce9b 100644 --- a/cloudkitty/tests/collectors/test_prometheus.py +++ b/cloudkitty/tests/collectors/test_prometheus.py @@ -24,7 +24,6 @@ from cloudkitty.common.prometheus_client import PrometheusResponseError from cloudkitty import dataframe from cloudkitty import tests from cloudkitty.tests import samples -from cloudkitty import transformer class PrometheusCollectorTest(tests.TestCase): @@ -53,8 +52,7 @@ class PrometheusCollectorTest(tests.TestCase): } } } - transformers = transformer.get_transformers() - self.collector = prometheus.PrometheusCollector(transformers, **args) + self.collector = prometheus.PrometheusCollector(**args) def test_fetch_all_build_query(self): query = ( diff --git a/cloudkitty/tests/transformers/__init__.py b/cloudkitty/tests/transformers/__init__.py deleted file mode 100644 index d27c86fa..00000000 --- a/cloudkitty/tests/transformers/__init__.py +++ /dev/null @@ -1,54 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2016 Objectif Libre -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -from cloudkitty.tests import samples -from cloudkitty import transformer - - -class Transformer(transformer.BaseTransformer): - compute_map = { - 'name': ['name', 'display_name'], - 'flavor': ['flavor', 'flavor.name', 'instance_type'], - 'vcpus': ['vcpus'], - 'memory': ['memory', 'memory_mb'], - 'image_id': ['image_id', 'image.id', 'image_meta.base_image_ref'], - 'availability_zone': [ - 'availability_zone', - 'OS-EXT-AZ.availability_zone'], - } - volume_map = { - 'volume_id': ['volume_id'], - 'name': ['display_name'], - 'availability_zone': ['availability_zone'], - 'size': ['size'], - } - test_map = {'test': lambda x, y: 'ok'} - - def _strip_network(self, res_metadata): - return {'test': 'ok'} - - -class TransformerMeta(Transformer): - metadata_item = 'metadata' - - -class EmptyClass(object): - pass - - -class ClassWithAttr(object): - def __init__(self, items=samples.COMPUTE_METADATA): - for key, val in items.items(): - setattr(self, key, val) diff --git a/cloudkitty/tests/transformers/test_base.py b/cloudkitty/tests/transformers/test_base.py deleted file mode 100644 index b46065c2..00000000 --- a/cloudkitty/tests/transformers/test_base.py +++ /dev/null @@ -1,68 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2016 Objectif Libre -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -import copy - -from cloudkitty import tests -from cloudkitty.tests import samples -from cloudkitty.tests import transformers as t_transformers - -TRANS_METADATA = { - 'availability_zone': 'nova', - 'flavor': 'm1.nano', - 'image_id': 'f5600101-8fa2-4864-899e-ebcb7ed6b568', - 'memory': '64', - 'name': 'prod1', - 'vcpus': '1'} - - -class TransformerBaseTest(tests.TestCase): - def test_strip_resource_on_dict(self): - metadata = copy.deepcopy(samples.COMPUTE_METADATA) - t_test = t_transformers.Transformer() - result = t_test.strip_resource_data('compute', metadata) - self.assertEqual(TRANS_METADATA, result) - - def test_strip_resource_with_no_rules(self): - metadata = copy.deepcopy(samples.COMPUTE_METADATA) - t_test = t_transformers.Transformer() - result = t_test.strip_resource_data('unknown', metadata) - self.assertEqual(samples.COMPUTE_METADATA, result) - - def test_strip_resource_with_func(self): - metadata = {'test': 'dummy'} - t_test = t_transformers.Transformer() - result = t_test.strip_resource_data('test', metadata) - self.assertEqual({'test': 'ok'}, result) - - def test_strip_resource_with_stripping_function(self): - metadata = {} - t_test = t_transformers.Transformer() - result = t_test.strip_resource_data('network', metadata) - self.assertEqual({'test': 'ok'}, result) - - def test_strip_resource_with_subitem(self): - test_obj = t_transformers.EmptyClass() - test_obj.metadata = copy.deepcopy(samples.COMPUTE_METADATA) - t_test = t_transformers.TransformerMeta() - result = t_test.strip_resource_data('compute', test_obj) - self.assertEqual(TRANS_METADATA, result) - - def test_strip_resource_with_attributes(self): - test_obj = t_transformers.EmptyClass() - test_obj.metadata = t_transformers.ClassWithAttr() - t_test = t_transformers.TransformerMeta() - result = t_test.strip_resource_data('compute', test_obj) - self.assertEqual(TRANS_METADATA, result) diff --git a/cloudkitty/transformer/__init__.py b/cloudkitty/transformer/__init__.py deleted file mode 100644 index dedc2121..00000000 --- a/cloudkitty/transformer/__init__.py +++ /dev/null @@ -1,69 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 Objectif Libre -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -import abc - -import six -from stevedore import extension - -TRANSFORMERS_NAMESPACE = 'cloudkitty.transformers' - - -def get_transformers(): - transformers = {} - transformer_exts = extension.ExtensionManager( - TRANSFORMERS_NAMESPACE, - invoke_on_load=True) - for transformer in transformer_exts: - t_name = transformer.name - t_obj = transformer.obj - transformers[t_name] = t_obj - return transformers - - -@six.add_metaclass(abc.ABCMeta) -class BaseTransformer(object): - metadata_item = '' - - def generic_strip(self, datatype, data): - metadata = getattr(data, self.metadata_item, data) - mappings = getattr(self, datatype + '_map', {}) - result = {} - for key, transform in mappings.items(): - if isinstance(transform, list): - for meta_key in transform: - if key not in result or result[key] is None: - try: - data = getattr(metadata, meta_key) - except AttributeError: - data = metadata.get(meta_key) - result[key] = data - else: - trans_data = transform(self, metadata) - if trans_data: - result[key] = trans_data - return result - - def strip_resource_data(self, res_type, res_data): - res_type = res_type.replace('.', '_') - strip_func = getattr(self, '_strip_' + res_type, None) - if strip_func: - return strip_func(res_data) - return self.generic_strip(res_type, res_data) or res_data - - def get_metadata(self, res_type): - """Return list of metadata available for given resource type.""" - - return [] diff --git a/cloudkitty/transformer/format.py b/cloudkitty/transformer/format.py deleted file mode 100644 index 196f431c..00000000 --- a/cloudkitty/transformer/format.py +++ /dev/null @@ -1,43 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 Objectif Libre -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# - -from oslo_log import log - -from cloudkitty import dataframe -from cloudkitty import transformer - - -LOG = log.getLogger(__name__) - - -class CloudKittyFormatTransformer(transformer.BaseTransformer): - def format_item(self, groupby, metadata, unit, qty=1.0): - # data = {} - # data['groupby'] = groupby - # data['metadata'] = metadata - # # For backward compatibility. - # data['desc'] = data['groupby'].copy() - # data['desc'].update(data['metadata']) - # data['vol'] = {'unit': unit, 'qty': qty} - - return dataframe.DataPoint(unit, qty, 0, groupby, metadata) - # return data - - def format_service(self, service, items): - data = {} - data[service] = items - - return data diff --git a/doc/source/developer/collector.rst b/doc/source/developer/collector.rst index 5c7ea7e4..260f2432 100644 --- a/doc/source/developer/collector.rst +++ b/doc/source/developer/collector.rst @@ -61,8 +61,8 @@ following prototype: .. autoclass:: cloudkitty.collector.BaseCollector :members: fetch_all -This method is supposed to return a list of objects formatted by -``CloudKittyFormatTransformer``. +This method is supposed to return a list of +``cloudkitty.dataframe.DataPoint`` objects. Example code of a basic collector: @@ -79,11 +79,12 @@ Example code of a basic collector: data = [] for CONDITION: # do stuff - data.append(self.t_cloudkitty.format_item( + data.append(dataframe.DataPoint( + unit, + qty, # int, float, decimal.Decimal or str + 0, # price groupby, # dict metadata, # dict - unit, # str - qty=qty, # int / float )) return data diff --git a/releasenotes/notes/remove-transformers-8d9949ed3088b055.yaml b/releasenotes/notes/remove-transformers-8d9949ed3088b055.yaml new file mode 100644 index 00000000..296628ef --- /dev/null +++ b/releasenotes/notes/remove-transformers-8d9949ed3088b055.yaml @@ -0,0 +1,5 @@ +--- +other: + - | + Since data frames are now represented as objects internally, transformers + are not used anymore and have been completely removed from the codebase. diff --git a/setup.cfg b/setup.cfg index c737c222..9f78287a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -57,9 +57,6 @@ cloudkitty.fetchers = gnocchi = cloudkitty.fetcher.gnocchi:GnocchiFetcher prometheus = cloudkitty.fetcher.prometheus:PrometheusFetcher -cloudkitty.transformers = - CloudKittyFormatTransformer = cloudkitty.transformer.format:CloudKittyFormatTransformer - cloudkitty.rating.processors = noop = cloudkitty.rating.noop:Noop hashmap = cloudkitty.rating.hash:HashMap