openstack
/
cloudkitty
Code Issues Proposed changes

Remove transformers from the codebase 

Since data frames are now handled as objects, transformers are no longer
required. This simplifies the global codebase.

Story: 2005890
Task: 36075
Change-Id: I76d9117bd95d80e51ca95804c999f145e65c3a2d
This commit is contained in:
Luka Peschke 2019-08-02 16:19:00 +02:00
parent c841ee8c29
commit 492ec063a7
15 changed files with 48 additions and 322 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
cloudkitty/collector/__init__.py
View File

@ -31,7 +31,6 @@ from voluptuous import Optional
from voluptuous import Required
from voluptuous import Schema
from cloudkitty import transformer
from cloudkitty import utils as ck_utils
@ -101,15 +100,12 @@ METRIC_BASE_SCHEMA = {
}
def get_collector(transformers=None):
def get_collector():
metrics_conf = ck_utils.load_conf(CONF.collect.metrics_conf)
if not transformers:
transformers = transformer.get_transformers()
collector_args = {
'period': CONF.collect.period,
'transformers': transformers,
'conf': metrics_conf,
}
collector_args.update({'conf': metrics_conf})
return driver.DriverManager(
COLLECTORS_NAMESPACE,
CONF.collect.collector,
@ -132,7 +128,6 @@ def get_metrics_based_collector_metadata():
Results are based on enabled collector and metrics in CONF.
"""
metrics_conf = ck_utils.load_conf(CONF.collect.metrics_conf)
transformers = transformer.get_transformers()
collector = get_collector_without_invoke()
metadata = {}
if 'metrics' in metrics_conf:
@ -140,23 +135,11 @@ def get_metrics_based_collector_metadata():
alt_name = metric.get('alt_name', metric_name)
metadata[alt_name] = collector.get_metadata(
metric_name,
transformers,
metrics_conf,
)
return metadata
class TransformerDependencyError(Exception):
"""Raised when a collector can't find a mandatory transformer."""
def __init__(self, collector, transformer):
super(TransformerDependencyError, self).__init__(
"Transformer '%s' not found, but required by %s" % (transformer,
collector))
self.collector = collector
self.transformer = transformer
class NoDataCollected(Exception):
"""Raised when the collection returned no data.
@ -173,11 +156,9 @@ class NoDataCollected(Exception):
@six.add_metaclass(abc.ABCMeta)
class BaseCollector(object):
collector_name = None
dependencies = ['CloudKittyFormatTransformer']
def __init__(self, transformers, **kwargs):
def __init__(self, **kwargs):
try:
self.transformers = transformers
self.period = kwargs['period']
self.conf = self.check_configuration(kwargs['conf'])
except KeyError as e:
@ -188,18 +169,6 @@ class BaseCollector(object):
LOG.error('Problem while checking configurations.', v)
raise v
self._check_transformers()
self.t_cloudkitty = self.transformers['CloudKittyFormatTransformer']
def _check_transformers(self):
"""Check for transformer prerequisites
"""
for dependency in self.dependencies:
if dependency not in self.transformers:
raise TransformerDependencyError(self.collector_name,
dependency)
@staticmethod
def check_configuration(conf):
"""Checks and validates metric configuration.
@ -229,7 +198,7 @@ class BaseCollector(object):
return trans_resource
@classmethod
def get_metadata(cls, resource_name, transformers):
def get_metadata(cls, resource_name):
"""Return metadata about collected resource as a dict.
Dict object should contain:
@ -247,8 +216,7 @@ class BaseCollector(object):
provided in the metric conf at initialization.
(Available in ``self.conf['groupby']`` and ``self.conf['metadata']``).
Returns a list of items formatted with
``CloudKittyFormatTransformer.format_item``.
Returns a list of cloudkitty.dataframe.DataPoint objects.
:param metric_name: Name of the metric to fetch
:type metric_name: str

20
cloudkitty/collector/gnocchi.py
View File

@ -29,6 +29,7 @@ from voluptuous import Required
from voluptuous import Schema
from cloudkitty import collector
from cloudkitty import dataframe
from cloudkitty import utils as ck_utils
@ -116,8 +117,8 @@ class GnocchiCollector(collector.BaseCollector):
collector_name = 'gnocchi'
def __init__(self, transformers, **kwargs):
super(GnocchiCollector, self).__init__(transformers, **kwargs)
def __init__(self, **kwargs):
super(GnocchiCollector, self).__init__(**kwargs)
adapter_options = {'connect_retries': 3}
if CONF.collector_gnocchi.gnocchi_auth_type == 'keystone':
@ -164,9 +165,8 @@ class GnocchiCollector(collector.BaseCollector):
return output
@classmethod
def get_metadata(cls, resource_name, transformers, conf):
info = super(GnocchiCollector, cls).get_metadata(resource_name,
transformers)
def get_metadata(cls, resource_name, conf):
info = super(GnocchiCollector, cls).get_metadata(resource_name)
try:
info["metadata"].extend(
conf[resource_name]['groupby']
@ -392,11 +392,11 @@ class GnocchiCollector(collector.BaseCollector):
project_id, start, end, e),
)
continue
data = self.t_cloudkitty.format_item(
formated_resources.append(dataframe.DataPoint(
met['unit'],
qty,
0,
groupby,
metadata,
met['unit'],
qty=qty,
)
formated_resources.append(data)
))
return formated_resources

27
cloudkitty/collector/monasca.py
View File

@ -25,7 +25,7 @@ from voluptuous import Required
from voluptuous import Schema
from cloudkitty import collector
from cloudkitty import transformer
from cloudkitty import dataframe
from cloudkitty import utils as ck_utils
@ -94,8 +94,8 @@ class MonascaCollector(collector.BaseCollector):
return output
def __init__(self, transformers, **kwargs):
super(MonascaCollector, self).__init__(transformers, **kwargs)
def __init__(self, **kwargs):
super(MonascaCollector, self).__init__(**kwargs)
self.auth = ks_loading.load_auth_from_conf_options(
CONF,
@ -129,7 +129,7 @@ class MonascaCollector(collector.BaseCollector):
return endpoint.url
return None
def _get_metadata(self, metric_name, transformers, conf):
def _get_metadata(self, metric_name, conf):
info = {}
info['unit'] = conf['metrics'][metric_name]['unit']
@ -141,12 +141,9 @@ class MonascaCollector(collector.BaseCollector):
# NOTE(lukapeschke) if anyone sees a better way to do this,
# please make a patch
@classmethod
def get_metadata(cls, resource_type, transformers, conf):
args = {
'transformers': transformer.get_transformers(),
'period': conf['period']}
tmp = cls(**args)
return tmp._get_metadata(resource_type, transformers, conf)
def get_metadata(cls, resource_type, conf):
tmp = cls(period=conf['period'])
return tmp._get_metadata(resource_type, conf)
def _get_dimensions(self, metric_name, project_id, q_filter):
dimensions = {}
@ -267,11 +264,11 @@ class MonascaCollector(collector.BaseCollector):
if len(d['statistics']):
metadata, groupby, qty = self._format_data(
met, d, resources_info)
data = self.t_cloudkitty.format_item(
formated_resources.append(dataframe.DataPoint(
met['unit'],
qty,
0,
groupby,
metadata,
met['unit'],
qty=qty,
)
formated_resources.append(data)
))
return formated_resources

16
cloudkitty/collector/prometheus.py
View File

@ -26,6 +26,7 @@ from cloudkitty import collector
from cloudkitty.collector.exceptions import CollectError
from cloudkitty.common.prometheus_client import PrometheusClient
from cloudkitty.common.prometheus_client import PrometheusResponseError
from cloudkitty import dataframe
from cloudkitty import utils as ck_utils
@ -76,8 +77,8 @@ PROMETHEUS_EXTRA_SCHEMA = {
class PrometheusCollector(collector.BaseCollector):
collector_name = 'prometheus'
def __init__(self, transformers, **kwargs):
super(PrometheusCollector, self).__init__(transformers, **kwargs)
def __init__(self, **kwargs):
super(PrometheusCollector, self).__init__(**kwargs)
url = CONF.collector_prometheus.prometheus_url
user = CONF.collector_prometheus.prometheus_user
@ -176,13 +177,12 @@ class PrometheusCollector(collector.BaseCollector):
item,
)
item = self.t_cloudkitty.format_item(
formatted_resources.append(dataframe.DataPoint(
self.conf[metric_name]['unit'],
qty,
0,
groupby,
metadata,
self.conf[metric_name]['unit'],
qty=qty,
)
formatted_resources.append(item)
))
return formatted_resources

4
cloudkitty/orchestrator.py
View File

@ -39,7 +39,6 @@ from cloudkitty import extension_manager
from cloudkitty import messaging
from cloudkitty import storage
from cloudkitty import storage_state as state
from cloudkitty import transformer
from cloudkitty import tzutils
from cloudkitty import utils as ck_utils
@ -334,8 +333,7 @@ class Orchestrator(cotyledon.Service):
invoke_on_load=True,
).driver
transformers = transformer.get_transformers()
self.collector = collector.get_collector(transformers)
self.collector = collector.get_collector()
self.storage = storage.get_storage()
self._state = state.StateManager()

2
cloudkitty/tests/collectors/test_gnocchi.py
View File

@ -17,7 +17,6 @@
from cloudkitty.collector import gnocchi
from cloudkitty import tests
from cloudkitty.tests import samples
from cloudkitty import transformer
class GnocchiCollectorTest(tests.TestCase):
@ -29,7 +28,6 @@ class GnocchiCollectorTest(tests.TestCase):
'gnocchi_auth_type', 'basic', 'collector_gnocchi')
self.collector = gnocchi.GnocchiCollector(
transformer.get_transformers(),
period=3600,
conf=samples.DEFAULT_METRICS_CONF,
)

2
cloudkitty/tests/collectors/test_monasca.py
View File

@ -18,7 +18,6 @@ import mock
from cloudkitty.collector import monasca as mon_collector
from cloudkitty import tests
from cloudkitty import transformer
class MonascaCollectorTest(tests.TestCase):
@ -50,7 +49,6 @@ class MonascaCollectorTest(tests.TestCase):
'MonascaCollector._get_monasca_endpoint',
return_value='http://noop'):
self.collector = mon_collector.MonascaCollector(
transformer.get_transformers(),
period=3600,
conf=conf,
)

4
cloudkitty/tests/collectors/test_prometheus.py
View File

@ -24,7 +24,6 @@ from cloudkitty.common.prometheus_client import PrometheusResponseError
from cloudkitty import dataframe
from cloudkitty import tests
from cloudkitty.tests import samples
from cloudkitty import transformer
class PrometheusCollectorTest(tests.TestCase):
@ -53,8 +52,7 @@ class PrometheusCollectorTest(tests.TestCase):
}
}
}
transformers = transformer.get_transformers()
self.collector = prometheus.PrometheusCollector(transformers, **args)
self.collector = prometheus.PrometheusCollector(**args)
def test_fetch_all_build_query(self):
query = (

54
cloudkitty/tests/transformers/__init__.py
View File

@ -1,54 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright 2016 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
from cloudkitty.tests import samples
from cloudkitty import transformer
class Transformer(transformer.BaseTransformer):
compute_map = {
'name': ['name', 'display_name'],
'flavor': ['flavor', 'flavor.name', 'instance_type'],
'vcpus': ['vcpus'],
'memory': ['memory', 'memory_mb'],
'image_id': ['image_id', 'image.id', 'image_meta.base_image_ref'],
'availability_zone': [
'availability_zone',
'OS-EXT-AZ.availability_zone'],
}
volume_map = {
'volume_id': ['volume_id'],
'name': ['display_name'],
'availability_zone': ['availability_zone'],
'size': ['size'],
}
test_map = {'test': lambda x, y: 'ok'}
def _strip_network(self, res_metadata):
return {'test': 'ok'}
class TransformerMeta(Transformer):
metadata_item = 'metadata'
class EmptyClass(object):
pass
class ClassWithAttr(object):
def __init__(self, items=samples.COMPUTE_METADATA):
for key, val in items.items():
setattr(self, key, val)

68
cloudkitty/tests/transformers/test_base.py
View File

@ -1,68 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright 2016 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import copy
from cloudkitty import tests
from cloudkitty.tests import samples
from cloudkitty.tests import transformers as t_transformers
TRANS_METADATA = {
'availability_zone': 'nova',
'flavor': 'm1.nano',
'image_id': 'f5600101-8fa2-4864-899e-ebcb7ed6b568',
'memory': '64',
'name': 'prod1',
'vcpus': '1'}
class TransformerBaseTest(tests.TestCase):
def test_strip_resource_on_dict(self):
metadata = copy.deepcopy(samples.COMPUTE_METADATA)
t_test = t_transformers.Transformer()
result = t_test.strip_resource_data('compute', metadata)
self.assertEqual(TRANS_METADATA, result)
def test_strip_resource_with_no_rules(self):
metadata = copy.deepcopy(samples.COMPUTE_METADATA)
t_test = t_transformers.Transformer()
result = t_test.strip_resource_data('unknown', metadata)
self.assertEqual(samples.COMPUTE_METADATA, result)
def test_strip_resource_with_func(self):
metadata = {'test': 'dummy'}
t_test = t_transformers.Transformer()
result = t_test.strip_resource_data('test', metadata)
self.assertEqual({'test': 'ok'}, result)
def test_strip_resource_with_stripping_function(self):
metadata = {}
t_test = t_transformers.Transformer()
result = t_test.strip_resource_data('network', metadata)
self.assertEqual({'test': 'ok'}, result)
def test_strip_resource_with_subitem(self):
test_obj = t_transformers.EmptyClass()
test_obj.metadata = copy.deepcopy(samples.COMPUTE_METADATA)
t_test = t_transformers.TransformerMeta()
result = t_test.strip_resource_data('compute', test_obj)
self.assertEqual(TRANS_METADATA, result)
def test_strip_resource_with_attributes(self):
test_obj = t_transformers.EmptyClass()
test_obj.metadata = t_transformers.ClassWithAttr()
t_test = t_transformers.TransformerMeta()
result = t_test.strip_resource_data('compute', test_obj)
self.assertEqual(TRANS_METADATA, result)

69
cloudkitty/transformer/__init__.py
View File

@ -1,69 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import abc
import six
from stevedore import extension
TRANSFORMERS_NAMESPACE = 'cloudkitty.transformers'
def get_transformers():
transformers = {}
transformer_exts = extension.ExtensionManager(
TRANSFORMERS_NAMESPACE,
invoke_on_load=True)
for transformer in transformer_exts:
t_name = transformer.name
t_obj = transformer.obj
transformers[t_name] = t_obj
return transformers
@six.add_metaclass(abc.ABCMeta)
class BaseTransformer(object):
metadata_item = ''
def generic_strip(self, datatype, data):
metadata = getattr(data, self.metadata_item, data)
mappings = getattr(self, datatype + '_map', {})
result = {}
for key, transform in mappings.items():
if isinstance(transform, list):
for meta_key in transform:
if key not in result or result[key] is None:
try:
data = getattr(metadata, meta_key)
except AttributeError:
data = metadata.get(meta_key)
result[key] = data
else:
trans_data = transform(self, metadata)
if trans_data:
result[key] = trans_data
return result
def strip_resource_data(self, res_type, res_data):
res_type = res_type.replace('.', '_')
strip_func = getattr(self, '_strip_' + res_type, None)
if strip_func:
return strip_func(res_data)
return self.generic_strip(res_type, res_data) or res_data
def get_metadata(self, res_type):
"""Return list of metadata available for given resource type."""
return []

43
cloudkitty/transformer/format.py
View File

@ -1,43 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
from oslo_log import log
from cloudkitty import dataframe
from cloudkitty import transformer
LOG = log.getLogger(__name__)
class CloudKittyFormatTransformer(transformer.BaseTransformer):
def format_item(self, groupby, metadata, unit, qty=1.0):
# data = {}
# data['groupby'] = groupby
# data['metadata'] = metadata
# # For backward compatibility.
# data['desc'] = data['groupby'].copy()
# data['desc'].update(data['metadata'])
# data['vol'] = {'unit': unit, 'qty': qty}
return dataframe.DataPoint(unit, qty, 0, groupby, metadata)
# return data
def format_service(self, service, items):
data = {}
data[service] = items
return data

11
doc/source/developer/collector.rst
View File

@ -61,8 +61,8 @@ following prototype:
.. autoclass:: cloudkitty.collector.BaseCollector
:members: fetch_all
This method is supposed to return a list of objects formatted by
``CloudKittyFormatTransformer``.
This method is supposed to return a list of
``cloudkitty.dataframe.DataPoint`` objects.
Example code of a basic collector:
@ -79,11 +79,12 @@ Example code of a basic collector:
data = []
for CONDITION:
# do stuff
data.append(self.t_cloudkitty.format_item(
data.append(dataframe.DataPoint(
unit,
qty, # int, float, decimal.Decimal or str
0, # price
groupby, # dict
metadata, # dict
unit, # str
qty=qty, # int / float
))
return data

5
releasenotes/notes/remove-transformers-8d9949ed3088b055.yaml Normal file
View File

@ -0,0 +1,5 @@
---
other:
- |
Since data frames are now represented as objects internally, transformers
are not used anymore and have been completely removed from the codebase.

3
setup.cfg
View File

@ -57,9 +57,6 @@ cloudkitty.fetchers =
gnocchi = cloudkitty.fetcher.gnocchi:GnocchiFetcher
prometheus = cloudkitty.fetcher.prometheus:PrometheusFetcher
cloudkitty.transformers =
CloudKittyFormatTransformer = cloudkitty.transformer.format:CloudKittyFormatTransformer
cloudkitty.rating.processors =
noop = cloudkitty.rating.noop:Noop
hashmap = cloudkitty.rating.hash:HashMap