openstack/cloudkitty
Code Issues Proposed changes

Remove transformers from the codebase

Browse Source 
Since data frames are now handled as objects, transformers are no longer
required. This simplifies the global codebase.

Story: 2005890
Task: 36075
Change-Id: I76d9117bd95d80e51ca95804c999f145e65c3a2d
This commit is contained in:
Luka Peschke 2019-08-02 16:19:00 +02:00
parent c841ee8c29
commit 492ec063a7
15 changed files with 48 additions and 322 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
cloudkitty/collector/__init__.py
View File

@ -31,7 +31,6 @@ from voluptuous import Optional
from voluptuous import Required
from voluptuous import Schema
from cloudkitty import transformer
from cloudkitty import utils as ck_utils
@ -101,15 +100,12 @@ METRIC_BASE_SCHEMA = {
}
def get_collector(transformers=None):
def get_collector():
metrics_conf = ck_utils.load_conf(CONF.collect.metrics_conf)
if not transformers:
transformers = transformer.get_transformers()
collector_args = {
'period': CONF.collect.period,
'transformers': transformers,
'conf': metrics_conf,
}
collector_args.update({'conf': metrics_conf})
return driver.DriverManager(
COLLECTORS_NAMESPACE,
CONF.collect.collector,
@ -132,7 +128,6 @@ def get_metrics_based_collector_metadata():
Results are based on enabled collector and metrics in CONF.
"""
metrics_conf = ck_utils.load_conf(CONF.collect.metrics_conf)
transformers = transformer.get_transformers()
collector = get_collector_without_invoke()
metadata = {}
if 'metrics' in metrics_conf:
@ -140,23 +135,11 @@ def get_metrics_based_collector_metadata():
alt_name = metric.get('alt_name', metric_name)
metadata[alt_name] = collector.get_metadata(
metric_name,
transformers,
metrics_conf,
)
return metadata
class TransformerDependencyError(Exception):
"""Raised when a collector can't find a mandatory transformer."""
def __init__(self, collector, transformer):
super(TransformerDependencyError, self).__init__(
"Transformer '%s' not found, but required by %s" % (transformer,
collector))
self.collector = collector
self.transformer = transformer
class NoDataCollected(Exception):
"""Raised when the collection returned no data.
@ -173,11 +156,9 @@ class NoDataCollected(Exception):
@six.add_metaclass(abc.ABCMeta)
class BaseCollector(object):
collector_name = None
dependencies = ['CloudKittyFormatTransformer']
def __init__(self, transformers, **kwargs):
def __init__(self, **kwargs):
try:
self.transformers = transformers
self.period = kwargs['period']
self.conf = self.check_configuration(kwargs['conf'])
except KeyError as e:
@ -188,18 +169,6 @@ class BaseCollector(object):
LOG.error('Problem while checking configurations.', v)
raise v
self._check_transformers()
self.t_cloudkitty = self.transformers['CloudKittyFormatTransformer']
def _check_transformers(self):
"""Check for transformer prerequisites
"""
for dependency in self.dependencies:
if dependency not in self.transformers:
raise TransformerDependencyError(self.collector_name,
dependency)
@staticmethod
def check_configuration(conf):
"""Checks and validates metric configuration.
@ -229,7 +198,7 @@ class BaseCollector(object):
return trans_resource
@classmethod
def get_metadata(cls, resource_name, transformers):
def get_metadata(cls, resource_name):
"""Return metadata about collected resource as a dict.
Dict object should contain:
@ -247,8 +216,7 @@ class BaseCollector(object):
provided in the metric conf at initialization.
(Available in ``self.conf['groupby']`` and ``self.conf['metadata']``).
Returns a list of items formatted with
``CloudKittyFormatTransformer.format_item``.
Returns a list of cloudkitty.dataframe.DataPoint objects.
:param metric_name: Name of the metric to fetch
:type metric_name: str

20
cloudkitty/collector/gnocchi.py
View File

@ -29,6 +29,7 @@ from voluptuous import Required
from voluptuous import Schema
from cloudkitty import collector
from cloudkitty import dataframe
from cloudkitty import utils as ck_utils
@ -116,8 +117,8 @@ class GnocchiCollector(collector.BaseCollector):
collector_name = 'gnocchi'
def __init__(self, transformers, **kwargs):
super(GnocchiCollector, self).__init__(transformers, **kwargs)
def __init__(self, **kwargs):
super(GnocchiCollector, self).__init__(**kwargs)
adapter_options = {'connect_retries': 3}
if CONF.collector_gnocchi.gnocchi_auth_type == 'keystone':
@ -164,9 +165,8 @@ class GnocchiCollector(collector.BaseCollector):
return output
@classmethod
def get_metadata(cls, resource_name, transformers, conf):
info = super(GnocchiCollector, cls).get_metadata(resource_name,
transformers)
def get_metadata(cls, resource_name, conf):
info = super(GnocchiCollector, cls).get_metadata(resource_name)
try:
info["metadata"].extend(
conf[resource_name]['groupby']
@ -392,11 +392,11 @@ class GnocchiCollector(collector.BaseCollector):
project_id, start, end, e),
)
continue
data = self.t_cloudkitty.format_item(
formated_resources.append(dataframe.DataPoint(
met['unit'],
qty,
0,
groupby,
metadata,
met['unit'],
qty=qty,
)
formated_resources.append(data)
))
return formated_resources

27
cloudkitty/collector/monasca.py
View File

@ -25,7 +25,7 @@ from voluptuous import Required
from voluptuous import Schema
from cloudkitty import collector
from cloudkitty import transformer
from cloudkitty import dataframe
from cloudkitty import utils as ck_utils
@ -94,8 +94,8 @@ class MonascaCollector(collector.BaseCollector):
return output
def __init__(self, transformers, **kwargs):
super(MonascaCollector, self).__init__(transformers, **kwargs)
def __init__(self, **kwargs):
super(MonascaCollector, self).__init__(**kwargs)
self.auth = ks_loading.load_auth_from_conf_options(
CONF,
@ -129,7 +129,7 @@ class MonascaCollector(collector.BaseCollector):
return endpoint.url
return None
def _get_metadata(self, metric_name, transformers, conf):
def _get_metadata(self, metric_name, conf):
info = {}
info['unit'] = conf['metrics'][metric_name]['unit']
@ -141,12 +141,9 @@ class MonascaCollector(collector.BaseCollector):
# NOTE(lukapeschke) if anyone sees a better way to do this,
# please make a patch
@classmethod
def get_metadata(cls, resource_type, transformers, conf):
args = {
'transformers': transformer.get_transformers(),
'period': conf['period']}
tmp = cls(**args)
return tmp._get_metadata(resource_type, transformers, conf)
def get_metadata(cls, resource_type, conf):
tmp = cls(period=conf['period'])
return tmp._get_metadata(resource_type, conf)
def _get_dimensions(self, metric_name, project_id, q_filter):
dimensions = {}
@ -267,11 +264,11 @@ class MonascaCollector(collector.BaseCollector):
if len(d['statistics']):
metadata, groupby, qty = self._format_data(
met, d, resources_info)
data = self.t_cloudkitty.format_item(
formated_resources.append(dataframe.DataPoint(
met['unit'],
qty,
0,
groupby,
metadata,
met['unit'],
qty=qty,
)
formated_resources.append(data)
))
return formated_resources

16
cloudkitty/collector/prometheus.py
View File

@ -26,6 +26,7 @@ from cloudkitty import collector
from cloudkitty.collector.exceptions import CollectError
from cloudkitty.common.prometheus_client import PrometheusClient
from cloudkitty.common.prometheus_client import PrometheusResponseError
from cloudkitty import dataframe
from cloudkitty import utils as ck_utils
@ -76,8 +77,8 @@ PROMETHEUS_EXTRA_SCHEMA = {
class PrometheusCollector(collector.BaseCollector):
collector_name = 'prometheus'
def __init__(self, transformers, **kwargs):
super(PrometheusCollector, self).__init__(transformers, **kwargs)
def __init__(self, **kwargs):
super(PrometheusCollector, self).__init__(**kwargs)
url = CONF.collector_prometheus.prometheus_url
user = CONF.collector_prometheus.prometheus_user
@ -176,13 +177,12 @@ class PrometheusCollector(collector.BaseCollector):
item,
)
item = self.t_cloudkitty.format_item(
formatted_resources.append(dataframe.DataPoint(
self.conf[metric_name]['unit'],
qty,
0,
groupby,
metadata,
self.conf[metric_name]['unit'],
qty=qty,
)
formatted_resources.append(item)
))
return formatted_resources

4
cloudkitty/orchestrator.py
View File

@ -39,7 +39,6 @@ from cloudkitty import extension_manager
from cloudkitty import messaging
from cloudkitty import storage
from cloudkitty import storage_state as state
from cloudkitty import transformer
from cloudkitty import tzutils
from cloudkitty import utils as ck_utils
@ -334,8 +333,7 @@ class Orchestrator(cotyledon.Service):
invoke_on_load=True,
).driver
transformers = transformer.get_transformers()
self.collector = collector.get_collector(transformers)
self.collector = collector.get_collector()
self.storage = storage.get_storage()
self._state = state.StateManager()

2
cloudkitty/tests/collectors/test_gnocchi.py
View File

@ -17,7 +17,6 @@
from cloudkitty.collector import gnocchi
from cloudkitty import tests
from cloudkitty.tests import samples
from cloudkitty import transformer
class GnocchiCollectorTest(tests.TestCase):
@ -29,7 +28,6 @@ class GnocchiCollectorTest(tests.TestCase):
'gnocchi_auth_type', 'basic', 'collector_gnocchi')
self.collector = gnocchi.GnocchiCollector(
transformer.get_transformers(),
period=3600,
conf=samples.DEFAULT_METRICS_CONF,
)

2
cloudkitty/tests/collectors/test_monasca.py
View File

@ -18,7 +18,6 @@ import mock
from cloudkitty.collector import monasca as mon_collector
from cloudkitty import tests
from cloudkitty import transformer
class MonascaCollectorTest(tests.TestCase):
@ -50,7 +49,6 @@ class MonascaCollectorTest(tests.TestCase):
'MonascaCollector._get_monasca_endpoint',
return_value='http://noop'):
self.collector = mon_collector.MonascaCollector(
transformer.get_transformers(),
period=3600,
conf=conf,
)

4
cloudkitty/tests/collectors/test_prometheus.py
View File

@ -24,7 +24,6 @@ from cloudkitty.common.prometheus_client import PrometheusResponseError
from cloudkitty import dataframe
from cloudkitty import tests
from cloudkitty.tests import samples
from cloudkitty import transformer
class PrometheusCollectorTest(tests.TestCase):
@ -53,8 +52,7 @@ class PrometheusCollectorTest(tests.TestCase):
}
}
}
transformers = transformer.get_transformers()
self.collector = prometheus.PrometheusCollector(transformers, **args)
self.collector = prometheus.PrometheusCollector(**args)
def test_fetch_all_build_query(self):
query = (

54
cloudkitty/tests/transformers/__init__.py
View File

@ -1,54 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright 2016 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
from cloudkitty.tests import samples
from cloudkitty import transformer
class Transformer(transformer.BaseTransformer):
compute_map = {
'name': ['name', 'display_name'],
'flavor': ['flavor', 'flavor.name', 'instance_type'],
'vcpus': ['vcpus'],
'memory': ['memory', 'memory_mb'],
'image_id': ['image_id', 'image.id', 'image_meta.base_image_ref'],
'availability_zone': [
'availability_zone',
'OS-EXT-AZ.availability_zone'],
}
volume_map = {
'volume_id': ['volume_id'],
'name': ['display_name'],
'availability_zone': ['availability_zone'],
'size': ['size'],
}
test_map = {'test': lambda x, y: 'ok'}
def _strip_network(self, res_metadata):
return {'test': 'ok'}
class TransformerMeta(Transformer):
metadata_item = 'metadata'
class EmptyClass(object):
pass
class ClassWithAttr(object):
def __init__(self, items=samples.COMPUTE_METADATA):
for key, val in items.items():
setattr(self, key, val)

68
cloudkitty/tests/transformers/test_base.py
View File

@ -1,68 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright 2016 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import copy
from cloudkitty import tests
from cloudkitty.tests import samples
from cloudkitty.tests import transformers as t_transformers
TRANS_METADATA = {
'availability_zone': 'nova',
'flavor': 'm1.nano',
'image_id': 'f5600101-8fa2-4864-899e-ebcb7ed6b568',
'memory': '64',
'name': 'prod1',
'vcpus': '1'}
class TransformerBaseTest(tests.TestCase):
def test_strip_resource_on_dict(self):
metadata = copy.deepcopy(samples.COMPUTE_METADATA)
t_test = t_transformers.Transformer()
result = t_test.strip_resource_data('compute', metadata)
self.assertEqual(TRANS_METADATA, result)
def test_strip_resource_with_no_rules(self):
metadata = copy.deepcopy(samples.COMPUTE_METADATA)
t_test = t_transformers.Transformer()
result = t_test.strip_resource_data('unknown', metadata)
self.assertEqual(samples.COMPUTE_METADATA, result)
def test_strip_resource_with_func(self):
metadata = {'test': 'dummy'}
t_test = t_transformers.Transformer()
result = t_test.strip_resource_data('test', metadata)
self.assertEqual({'test': 'ok'}, result)
def test_strip_resource_with_stripping_function(self):
metadata = {}
t_test = t_transformers.Transformer()
result = t_test.strip_resource_data('network', metadata)
self.assertEqual({'test': 'ok'}, result)
def test_strip_resource_with_subitem(self):
test_obj = t_transformers.EmptyClass()
test_obj.metadata = copy.deepcopy(samples.COMPUTE_METADATA)
t_test = t_transformers.TransformerMeta()
result = t_test.strip_resource_data('compute', test_obj)
self.assertEqual(TRANS_METADATA, result)
def test_strip_resource_with_attributes(self):
test_obj = t_transformers.EmptyClass()
test_obj.metadata = t_transformers.ClassWithAttr()
t_test = t_transformers.TransformerMeta()
result = t_test.strip_resource_data('compute', test_obj)
self.assertEqual(TRANS_METADATA, result)

69
cloudkitty/transformer/__init__.py
View File

@ -1,69 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import abc
import six
from stevedore import extension
TRANSFORMERS_NAMESPACE = 'cloudkitty.transformers'
def get_transformers():
transformers = {}
transformer_exts = extension.ExtensionManager(
TRANSFORMERS_NAMESPACE,
invoke_on_load=True)
for transformer in transformer_exts:
t_name = transformer.name
t_obj = transformer.obj
transformers[t_name] = t_obj
return transformers
@six.add_metaclass(abc.ABCMeta)
class BaseTransformer(object):
metadata_item = ''
def generic_strip(self, datatype, data):
metadata = getattr(data, self.metadata_item, data)
mappings = getattr(self, datatype + '_map', {})
result = {}
for key, transform in mappings.items():
if isinstance(transform, list):
for meta_key in transform:
if key not in result or result[key] is None:
try:
data = getattr(metadata, meta_key)
except AttributeError:
data = metadata.get(meta_key)
result[key] = data
else:
trans_data = transform(self, metadata)
if trans_data:
result[key] = trans_data
return result
def strip_resource_data(self, res_type, res_data):
res_type = res_type.replace('.', '_')
strip_func = getattr(self, '_strip_' + res_type, None)
if strip_func:
return strip_func(res_data)
return self.generic_strip(res_type, res_data) or res_data
def get_metadata(self, res_type):
"""Return list of metadata available for given resource type."""
return []

43
cloudkitty/transformer/format.py
View File

@ -1,43 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
from oslo_log import log
from cloudkitty import dataframe
from cloudkitty import transformer
LOG = log.getLogger(__name__)
class CloudKittyFormatTransformer(transformer.BaseTransformer):
def format_item(self, groupby, metadata, unit, qty=1.0):
# data = {}
# data['groupby'] = groupby
# data['metadata'] = metadata
# # For backward compatibility.
# data['desc'] = data['groupby'].copy()
# data['desc'].update(data['metadata'])
# data['vol'] = {'unit': unit, 'qty': qty}
return dataframe.DataPoint(unit, qty, 0, groupby, metadata)
# return data
def format_service(self, service, items):
data = {}
data[service] = items
return data

11
doc/source/developer/collector.rst
View File

@ -61,8 +61,8 @@ following prototype:
.. autoclass:: cloudkitty.collector.BaseCollector
:members: fetch_all
This method is supposed to return a list of objects formatted by
``CloudKittyFormatTransformer``.
This method is supposed to return a list of
``cloudkitty.dataframe.DataPoint`` objects.
Example code of a basic collector:
@ -79,11 +79,12 @@ Example code of a basic collector:
data = []
for CONDITION:
# do stuff
data.append(self.t_cloudkitty.format_item(
data.append(dataframe.DataPoint(
unit,
qty, # int, float, decimal.Decimal or str
0, # price
groupby, # dict
metadata, # dict
unit, # str
qty=qty, # int / float
))
return data

5
releasenotes/notes/remove-transformers-8d9949ed3088b055.yaml Normal file
View File

@ -0,0 +1,5 @@
---
other:
- |
Since data frames are now represented as objects internally, transformers
are not used anymore and have been completely removed from the codebase.

3
setup.cfg
View File

@ -57,9 +57,6 @@ cloudkitty.fetchers =
gnocchi = cloudkitty.fetcher.gnocchi:GnocchiFetcher
prometheus = cloudkitty.fetcher.prometheus:PrometheusFetcher
cloudkitty.transformers =
CloudKittyFormatTransformer = cloudkitty.transformer.format:CloudKittyFormatTransformer
cloudkitty.rating.processors =
noop = cloudkitty.rating.noop:Noop
hashmap = cloudkitty.rating.hash:HashMap