Browse Source

Remove transformers from the codebase

Since data frames are now handled as objects, transformers are no longer
required. This simplifies the global codebase.

Story: 2005890
Task: 36075
Change-Id: I76d9117bd95d80e51ca95804c999f145e65c3a2d
tags/11.0.0
Luka Peschke 6 months ago
parent
commit
492ec063a7
15 changed files with 48 additions and 322 deletions
  1. +5
    -37
      cloudkitty/collector/__init__.py
  2. +10
    -10
      cloudkitty/collector/gnocchi.py
  3. +12
    -15
      cloudkitty/collector/monasca.py
  4. +8
    -8
      cloudkitty/collector/prometheus.py
  5. +1
    -3
      cloudkitty/orchestrator.py
  6. +0
    -2
      cloudkitty/tests/collectors/test_gnocchi.py
  7. +0
    -2
      cloudkitty/tests/collectors/test_monasca.py
  8. +1
    -3
      cloudkitty/tests/collectors/test_prometheus.py
  9. +0
    -54
      cloudkitty/tests/transformers/__init__.py
  10. +0
    -68
      cloudkitty/tests/transformers/test_base.py
  11. +0
    -69
      cloudkitty/transformer/__init__.py
  12. +0
    -43
      cloudkitty/transformer/format.py
  13. +6
    -5
      doc/source/developer/collector.rst
  14. +5
    -0
      releasenotes/notes/remove-transformers-8d9949ed3088b055.yaml
  15. +0
    -3
      setup.cfg

+ 5
- 37
cloudkitty/collector/__init__.py View File

@@ -31,7 +31,6 @@ from voluptuous import Optional
from voluptuous import Required
from voluptuous import Schema

from cloudkitty import transformer
from cloudkitty import utils as ck_utils


@@ -101,15 +100,12 @@ METRIC_BASE_SCHEMA = {
}


def get_collector(transformers=None):
def get_collector():
metrics_conf = ck_utils.load_conf(CONF.collect.metrics_conf)
if not transformers:
transformers = transformer.get_transformers()
collector_args = {
'period': CONF.collect.period,
'transformers': transformers,
'conf': metrics_conf,
}
collector_args.update({'conf': metrics_conf})
return driver.DriverManager(
COLLECTORS_NAMESPACE,
CONF.collect.collector,
@@ -132,7 +128,6 @@ def get_metrics_based_collector_metadata():
Results are based on enabled collector and metrics in CONF.
"""
metrics_conf = ck_utils.load_conf(CONF.collect.metrics_conf)
transformers = transformer.get_transformers()
collector = get_collector_without_invoke()
metadata = {}
if 'metrics' in metrics_conf:
@@ -140,23 +135,11 @@ def get_metrics_based_collector_metadata():
alt_name = metric.get('alt_name', metric_name)
metadata[alt_name] = collector.get_metadata(
metric_name,
transformers,
metrics_conf,
)
return metadata


class TransformerDependencyError(Exception):
"""Raised when a collector can't find a mandatory transformer."""

def __init__(self, collector, transformer):
super(TransformerDependencyError, self).__init__(
"Transformer '%s' not found, but required by %s" % (transformer,
collector))
self.collector = collector
self.transformer = transformer


class NoDataCollected(Exception):
"""Raised when the collection returned no data.

@@ -173,11 +156,9 @@ class NoDataCollected(Exception):
@six.add_metaclass(abc.ABCMeta)
class BaseCollector(object):
collector_name = None
dependencies = ['CloudKittyFormatTransformer']

def __init__(self, transformers, **kwargs):
def __init__(self, **kwargs):
try:
self.transformers = transformers
self.period = kwargs['period']
self.conf = self.check_configuration(kwargs['conf'])
except KeyError as e:
@@ -188,18 +169,6 @@ class BaseCollector(object):
LOG.error('Problem while checking configurations.', v)
raise v

self._check_transformers()
self.t_cloudkitty = self.transformers['CloudKittyFormatTransformer']

def _check_transformers(self):
"""Check for transformer prerequisites

"""
for dependency in self.dependencies:
if dependency not in self.transformers:
raise TransformerDependencyError(self.collector_name,
dependency)

@staticmethod
def check_configuration(conf):
"""Checks and validates metric configuration.
@@ -229,7 +198,7 @@ class BaseCollector(object):
return trans_resource

@classmethod
def get_metadata(cls, resource_name, transformers):
def get_metadata(cls, resource_name):
"""Return metadata about collected resource as a dict.

Dict object should contain:
@@ -247,8 +216,7 @@ class BaseCollector(object):
provided in the metric conf at initialization.
(Available in ``self.conf['groupby']`` and ``self.conf['metadata']``).

Returns a list of items formatted with
``CloudKittyFormatTransformer.format_item``.
Returns a list of cloudkitty.dataframe.DataPoint objects.

:param metric_name: Name of the metric to fetch
:type metric_name: str

+ 10
- 10
cloudkitty/collector/gnocchi.py View File

@@ -29,6 +29,7 @@ from voluptuous import Required
from voluptuous import Schema

from cloudkitty import collector
from cloudkitty import dataframe
from cloudkitty import utils as ck_utils


@@ -116,8 +117,8 @@ class GnocchiCollector(collector.BaseCollector):

collector_name = 'gnocchi'

def __init__(self, transformers, **kwargs):
super(GnocchiCollector, self).__init__(transformers, **kwargs)
def __init__(self, **kwargs):
super(GnocchiCollector, self).__init__(**kwargs)

adapter_options = {'connect_retries': 3}
if CONF.collector_gnocchi.gnocchi_auth_type == 'keystone':
@@ -164,9 +165,8 @@ class GnocchiCollector(collector.BaseCollector):
return output

@classmethod
def get_metadata(cls, resource_name, transformers, conf):
info = super(GnocchiCollector, cls).get_metadata(resource_name,
transformers)
def get_metadata(cls, resource_name, conf):
info = super(GnocchiCollector, cls).get_metadata(resource_name)
try:
info["metadata"].extend(
conf[resource_name]['groupby']
@@ -392,11 +392,11 @@ class GnocchiCollector(collector.BaseCollector):
project_id, start, end, e),
)
continue
data = self.t_cloudkitty.format_item(
formated_resources.append(dataframe.DataPoint(
met['unit'],
qty,
0,
groupby,
metadata,
met['unit'],
qty=qty,
)
formated_resources.append(data)
))
return formated_resources

+ 12
- 15
cloudkitty/collector/monasca.py View File

@@ -25,7 +25,7 @@ from voluptuous import Required
from voluptuous import Schema

from cloudkitty import collector
from cloudkitty import transformer
from cloudkitty import dataframe
from cloudkitty import utils as ck_utils


@@ -94,8 +94,8 @@ class MonascaCollector(collector.BaseCollector):

return output

def __init__(self, transformers, **kwargs):
super(MonascaCollector, self).__init__(transformers, **kwargs)
def __init__(self, **kwargs):
super(MonascaCollector, self).__init__(**kwargs)

self.auth = ks_loading.load_auth_from_conf_options(
CONF,
@@ -129,7 +129,7 @@ class MonascaCollector(collector.BaseCollector):
return endpoint.url
return None

def _get_metadata(self, metric_name, transformers, conf):
def _get_metadata(self, metric_name, conf):
info = {}
info['unit'] = conf['metrics'][metric_name]['unit']

@@ -141,12 +141,9 @@ class MonascaCollector(collector.BaseCollector):
# NOTE(lukapeschke) if anyone sees a better way to do this,
# please make a patch
@classmethod
def get_metadata(cls, resource_type, transformers, conf):
args = {
'transformers': transformer.get_transformers(),
'period': conf['period']}
tmp = cls(**args)
return tmp._get_metadata(resource_type, transformers, conf)
def get_metadata(cls, resource_type, conf):
tmp = cls(period=conf['period'])
return tmp._get_metadata(resource_type, conf)

def _get_dimensions(self, metric_name, project_id, q_filter):
dimensions = {}
@@ -267,11 +264,11 @@ class MonascaCollector(collector.BaseCollector):
if len(d['statistics']):
metadata, groupby, qty = self._format_data(
met, d, resources_info)
data = self.t_cloudkitty.format_item(
formated_resources.append(dataframe.DataPoint(
met['unit'],
qty,
0,
groupby,
metadata,
met['unit'],
qty=qty,
)
formated_resources.append(data)
))
return formated_resources

+ 8
- 8
cloudkitty/collector/prometheus.py View File

@@ -26,6 +26,7 @@ from cloudkitty import collector
from cloudkitty.collector.exceptions import CollectError
from cloudkitty.common.prometheus_client import PrometheusClient
from cloudkitty.common.prometheus_client import PrometheusResponseError
from cloudkitty import dataframe
from cloudkitty import utils as ck_utils


@@ -76,8 +77,8 @@ PROMETHEUS_EXTRA_SCHEMA = {
class PrometheusCollector(collector.BaseCollector):
collector_name = 'prometheus'

def __init__(self, transformers, **kwargs):
super(PrometheusCollector, self).__init__(transformers, **kwargs)
def __init__(self, **kwargs):
super(PrometheusCollector, self).__init__(**kwargs)
url = CONF.collector_prometheus.prometheus_url

user = CONF.collector_prometheus.prometheus_user
@@ -176,13 +177,12 @@ class PrometheusCollector(collector.BaseCollector):
item,
)

item = self.t_cloudkitty.format_item(
formatted_resources.append(dataframe.DataPoint(
self.conf[metric_name]['unit'],
qty,
0,
groupby,
metadata,
self.conf[metric_name]['unit'],
qty=qty,
)

formatted_resources.append(item)
))

return formatted_resources

+ 1
- 3
cloudkitty/orchestrator.py View File

@@ -39,7 +39,6 @@ from cloudkitty import extension_manager
from cloudkitty import messaging
from cloudkitty import storage
from cloudkitty import storage_state as state
from cloudkitty import transformer
from cloudkitty import tzutils
from cloudkitty import utils as ck_utils

@@ -334,8 +333,7 @@ class Orchestrator(cotyledon.Service):
invoke_on_load=True,
).driver

transformers = transformer.get_transformers()
self.collector = collector.get_collector(transformers)
self.collector = collector.get_collector()
self.storage = storage.get_storage()
self._state = state.StateManager()


+ 0
- 2
cloudkitty/tests/collectors/test_gnocchi.py View File

@@ -17,7 +17,6 @@
from cloudkitty.collector import gnocchi
from cloudkitty import tests
from cloudkitty.tests import samples
from cloudkitty import transformer


class GnocchiCollectorTest(tests.TestCase):
@@ -29,7 +28,6 @@ class GnocchiCollectorTest(tests.TestCase):
'gnocchi_auth_type', 'basic', 'collector_gnocchi')

self.collector = gnocchi.GnocchiCollector(
transformer.get_transformers(),
period=3600,
conf=samples.DEFAULT_METRICS_CONF,
)

+ 0
- 2
cloudkitty/tests/collectors/test_monasca.py View File

@@ -18,7 +18,6 @@ import mock

from cloudkitty.collector import monasca as mon_collector
from cloudkitty import tests
from cloudkitty import transformer


class MonascaCollectorTest(tests.TestCase):
@@ -50,7 +49,6 @@ class MonascaCollectorTest(tests.TestCase):
'MonascaCollector._get_monasca_endpoint',
return_value='http://noop'):
self.collector = mon_collector.MonascaCollector(
transformer.get_transformers(),
period=3600,
conf=conf,
)

+ 1
- 3
cloudkitty/tests/collectors/test_prometheus.py View File

@@ -24,7 +24,6 @@ from cloudkitty.common.prometheus_client import PrometheusResponseError
from cloudkitty import dataframe
from cloudkitty import tests
from cloudkitty.tests import samples
from cloudkitty import transformer


class PrometheusCollectorTest(tests.TestCase):
@@ -53,8 +52,7 @@ class PrometheusCollectorTest(tests.TestCase):
}
}
}
transformers = transformer.get_transformers()
self.collector = prometheus.PrometheusCollector(transformers, **args)
self.collector = prometheus.PrometheusCollector(**args)

def test_fetch_all_build_query(self):
query = (

+ 0
- 54
cloudkitty/tests/transformers/__init__.py View File

@@ -1,54 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright 2016 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
from cloudkitty.tests import samples
from cloudkitty import transformer


class Transformer(transformer.BaseTransformer):
compute_map = {
'name': ['name', 'display_name'],
'flavor': ['flavor', 'flavor.name', 'instance_type'],
'vcpus': ['vcpus'],
'memory': ['memory', 'memory_mb'],
'image_id': ['image_id', 'image.id', 'image_meta.base_image_ref'],
'availability_zone': [
'availability_zone',
'OS-EXT-AZ.availability_zone'],
}
volume_map = {
'volume_id': ['volume_id'],
'name': ['display_name'],
'availability_zone': ['availability_zone'],
'size': ['size'],
}
test_map = {'test': lambda x, y: 'ok'}

def _strip_network(self, res_metadata):
return {'test': 'ok'}


class TransformerMeta(Transformer):
metadata_item = 'metadata'


class EmptyClass(object):
pass


class ClassWithAttr(object):
def __init__(self, items=samples.COMPUTE_METADATA):
for key, val in items.items():
setattr(self, key, val)

+ 0
- 68
cloudkitty/tests/transformers/test_base.py View File

@@ -1,68 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright 2016 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import copy

from cloudkitty import tests
from cloudkitty.tests import samples
from cloudkitty.tests import transformers as t_transformers

TRANS_METADATA = {
'availability_zone': 'nova',
'flavor': 'm1.nano',
'image_id': 'f5600101-8fa2-4864-899e-ebcb7ed6b568',
'memory': '64',
'name': 'prod1',
'vcpus': '1'}


class TransformerBaseTest(tests.TestCase):
def test_strip_resource_on_dict(self):
metadata = copy.deepcopy(samples.COMPUTE_METADATA)
t_test = t_transformers.Transformer()
result = t_test.strip_resource_data('compute', metadata)
self.assertEqual(TRANS_METADATA, result)

def test_strip_resource_with_no_rules(self):
metadata = copy.deepcopy(samples.COMPUTE_METADATA)
t_test = t_transformers.Transformer()
result = t_test.strip_resource_data('unknown', metadata)
self.assertEqual(samples.COMPUTE_METADATA, result)

def test_strip_resource_with_func(self):
metadata = {'test': 'dummy'}
t_test = t_transformers.Transformer()
result = t_test.strip_resource_data('test', metadata)
self.assertEqual({'test': 'ok'}, result)

def test_strip_resource_with_stripping_function(self):
metadata = {}
t_test = t_transformers.Transformer()
result = t_test.strip_resource_data('network', metadata)
self.assertEqual({'test': 'ok'}, result)

def test_strip_resource_with_subitem(self):
test_obj = t_transformers.EmptyClass()
test_obj.metadata = copy.deepcopy(samples.COMPUTE_METADATA)
t_test = t_transformers.TransformerMeta()
result = t_test.strip_resource_data('compute', test_obj)
self.assertEqual(TRANS_METADATA, result)

def test_strip_resource_with_attributes(self):
test_obj = t_transformers.EmptyClass()
test_obj.metadata = t_transformers.ClassWithAttr()
t_test = t_transformers.TransformerMeta()
result = t_test.strip_resource_data('compute', test_obj)
self.assertEqual(TRANS_METADATA, result)

+ 0
- 69
cloudkitty/transformer/__init__.py View File

@@ -1,69 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import abc

import six
from stevedore import extension

TRANSFORMERS_NAMESPACE = 'cloudkitty.transformers'


def get_transformers():
transformers = {}
transformer_exts = extension.ExtensionManager(
TRANSFORMERS_NAMESPACE,
invoke_on_load=True)
for transformer in transformer_exts:
t_name = transformer.name
t_obj = transformer.obj
transformers[t_name] = t_obj
return transformers


@six.add_metaclass(abc.ABCMeta)
class BaseTransformer(object):
metadata_item = ''

def generic_strip(self, datatype, data):
metadata = getattr(data, self.metadata_item, data)
mappings = getattr(self, datatype + '_map', {})
result = {}
for key, transform in mappings.items():
if isinstance(transform, list):
for meta_key in transform:
if key not in result or result[key] is None:
try:
data = getattr(metadata, meta_key)
except AttributeError:
data = metadata.get(meta_key)
result[key] = data
else:
trans_data = transform(self, metadata)
if trans_data:
result[key] = trans_data
return result

def strip_resource_data(self, res_type, res_data):
res_type = res_type.replace('.', '_')
strip_func = getattr(self, '_strip_' + res_type, None)
if strip_func:
return strip_func(res_data)
return self.generic_strip(res_type, res_data) or res_data

def get_metadata(self, res_type):
"""Return list of metadata available for given resource type."""

return []

+ 0
- 43
cloudkitty/transformer/format.py View File

@@ -1,43 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#

from oslo_log import log

from cloudkitty import dataframe
from cloudkitty import transformer


LOG = log.getLogger(__name__)


class CloudKittyFormatTransformer(transformer.BaseTransformer):
def format_item(self, groupby, metadata, unit, qty=1.0):
# data = {}
# data['groupby'] = groupby
# data['metadata'] = metadata
# # For backward compatibility.
# data['desc'] = data['groupby'].copy()
# data['desc'].update(data['metadata'])
# data['vol'] = {'unit': unit, 'qty': qty}

return dataframe.DataPoint(unit, qty, 0, groupby, metadata)
# return data

def format_service(self, service, items):
data = {}
data[service] = items

return data

+ 6
- 5
doc/source/developer/collector.rst View File

@@ -61,8 +61,8 @@ following prototype:
.. autoclass:: cloudkitty.collector.BaseCollector
:members: fetch_all

This method is supposed to return a list of objects formatted by
``CloudKittyFormatTransformer``.
This method is supposed to return a list of
``cloudkitty.dataframe.DataPoint`` objects.

Example code of a basic collector:

@@ -79,11 +79,12 @@ Example code of a basic collector:
data = []
for CONDITION:
# do stuff
data.append(self.t_cloudkitty.format_item(
data.append(dataframe.DataPoint(
unit,
qty, # int, float, decimal.Decimal or str
0, # price
groupby, # dict
metadata, # dict
unit, # str
qty=qty, # int / float
))

return data

+ 5
- 0
releasenotes/notes/remove-transformers-8d9949ed3088b055.yaml View File

@@ -0,0 +1,5 @@
---
other:
- |
Since data frames are now represented as objects internally, transformers
are not used anymore and have been completely removed from the codebase.

+ 0
- 3
setup.cfg View File

@@ -57,9 +57,6 @@ cloudkitty.fetchers =
gnocchi = cloudkitty.fetcher.gnocchi:GnocchiFetcher
prometheus = cloudkitty.fetcher.prometheus:PrometheusFetcher

cloudkitty.transformers =
CloudKittyFormatTransformer = cloudkitty.transformer.format:CloudKittyFormatTransformer

cloudkitty.rating.processors =
noop = cloudkitty.rating.noop:Noop
hashmap = cloudkitty.rating.hash:HashMap

Loading…
Cancel
Save