add transformers for trove

slightly alter how config is loaded to transformers.

Change-Id: I32bf34d270604e8c755160a7dac71e99a38f2a74

distil/collector/base.py

@@ -205,7 +205,9 @@ class BaseCollector(object):
         service = (mapping['service'] if 'service' in mapping
                    else mapping['meter'])
 
-        transformer = d_transformer.get_transformer(mapping['transformer'])
+        transformer = d_transformer.get_transformer(
+            mapping['transformer'],
+            override_config=mapping.get('transformer_config', {}))
 
         for res_id, entries in usage_by_resource.items():
             transformed = transformer.transform_usage(

distil/common/general.py

@@ -33,7 +33,7 @@ LOG = logging.getLogger(__name__)
 _TRANS_CONFIG = None
 
 
-def get_transformer_config():
+def get_transformer_config(name):
     global _TRANS_CONFIG
 
     if not _TRANS_CONFIG:
@@ -43,7 +43,7 @@ def get_transformer_config():
         except IOError as e:
             raise e
 
-    return _TRANS_CONFIG
+    return _TRANS_CONFIG.get(name, {})
 
 
 def get_windows(start, end):

distil/common/openstack.py

@@ -17,12 +17,14 @@ import re
 
 from ceilometerclient import client as ceilometerclient
 from cinderclient.v2 import client as cinderclient
+from cinderclient.exceptions import NotFound as CinderNotFound
 from glanceclient import client as glanceclient
 from keystoneauth1.identity import v3
 from keystoneauth1.exceptions import NotFound
 from keystoneauth1 import session
 from keystoneclient.v3 import client as ks_client
 from novaclient import client as novaclient
+from novaclient.exceptions import NotFound as NovaNotFound
 from oslo_config import cfg
 
 from distil.common import cache as distil_cache
@@ -30,7 +32,7 @@ from distil.common import general
 
 CONF = cfg.CONF
 KS_SESSION = None
-cache = defaultdict(list)
+cache = defaultdict(dict)
 ROOT_DEVICE_PATTERN = re.compile('^/dev/(x?v|s|h)da1?$')
 
 
@@ -163,20 +165,43 @@ def get_root_volume(instance_id):
 
 
 @general.disable_ssl_warnings
-def get_volume_type(volume_type):
-    if not cache.get("volume_types"):
-        cinder = get_cinder_client()
-        for vtype in cinder.volume_types.list():
-            cache['volume_types'].append({'id': vtype.id, 'name': vtype.name})
-
-    for vtype in cache['volume_types']:
-        # check name first, as that will be more common
-        if vtype['name'] == volume_type:
-            return volume_type
-        elif vtype['id'] == volume_type:
-            return vtype['name']
-
+def get_flavor_name(flavor_id):
+    if flavor_id not in cache["flavors"]:
+        nova = get_nova_client()
+        try:
+            flavor_name = nova.flavors.get(flavor_id).name
+        except NovaNotFound:
             return None
+        cache["flavors"][flavor_id] = flavor_name
+    return cache["flavors"][flavor_id]
+
+
+@general.disable_ssl_warnings
+def get_volume_type_for_volume(volume_id):
+    if volume_id not in cache["volume_id_to_type"]:
+        cinder = get_cinder_client()
+        try:
+            vol = cinder.volumes.get(volume_id)
+        except CinderNotFound:
+            return None
+        cache["volume_id_to_type"][volume_id] = vol.volume_type
+    return cache["volume_id_to_type"][volume_id]
+
+
+@general.disable_ssl_warnings
+def get_volume_type_name(volume_type):
+    if volume_type not in cache["volume_types"]:
+        cinder = get_cinder_client()
+        try:
+            vtype = cinder.volume_types.get(volume_type)
+        except CinderNotFound:
+            try:
+                vtype = cinder.volume_types.find(name=volume_type)
+            except CinderNotFound:
+                return None
+        cache["volume_types"][vtype.id] = vtype.name
+        cache["volume_types"][vtype.name] = vtype.name
+    return cache["volume_types"][volume_type]
 
 
 @general.disable_ssl_warnings

distil/transformer/__init__.py

@@ -20,8 +20,10 @@ from distil.common import general
 
 class BaseTransformer(object):
 
-    def __init__(self, *args, **kwargs):
-        self.config = general.get_transformer_config()
+    def __init__(self, name, override_config=None):
+        self.config = general.get_transformer_config(name)
+        if override_config:
+            self.config.update(override_config)
 
     def transform_usage(self, meter_name, raw_data, start_at, end_at):
         return self._transform_usage(meter_name, raw_data, start_at, end_at)
@@ -35,5 +37,6 @@ def get_transformer(name, **kwargs):
         'distil.transformer',
         name,
         invoke_on_load=True,
+        invoke_args=(name,),
         invoke_kwds=kwargs
     ).driver

distil/transformer/arithmetic.py

@@ -59,7 +59,7 @@ class BlockStorageMaxTransformer(MaxTransformer):
 
         if "volume_type" in data[-1]['metadata']:
             vtype = data[-1]['metadata']['volume_type']
-            service = openstack.get_volume_type(vtype)
+            service = openstack.get_volume_type_name(vtype)
             if not service:
                 service = name
         else:
@@ -69,6 +69,30 @@ class BlockStorageMaxTransformer(MaxTransformer):
         return {service: max_vol * hours}
 
 
+class DatabaseVolumeMaxTransformer(BaseTransformer):
+    """
+    Variantion on the GaugeMax Transformer that checks for
+    volume_type and uses that as the service, or uses the
+    default service name.
+
+    It also gets the actual volume size from metadata.
+    """
+
+    def _transform_usage(self, name, data, start, end):
+        if not data:
+            return None
+
+        max_vol = max([int(v["metadata"]["volume.size"]) for v in data])
+
+        volume_type = openstack.get_volume_type_for_volume(
+            data[-1]['metadata']['volume_id'])
+        if not volume_type:
+            return None
+
+        hours = (end - start).total_seconds() / 3600.0
+        return {volume_type: max_vol * hours}
+
+
 class ObjectStorageMaxTransformer(MaxTransformer):
     """
     Variantion on the GaugeMax Transformer that checks for

distil/transformer/conversion.py

@@ -40,7 +40,7 @@ class UpTimeTransformer(BaseTransformer):
 
     def _transform_usage(self, name, data, start, end):
         # get tracked states from config
-        tracked = self.config['uptime']['tracked_states']
+        tracked = self.config['tracked_states']
 
         usage_dict = {}
 
@@ -118,10 +118,10 @@ class FromImageTransformer(BaseTransformer):
     """
 
     def _transform_usage(self, name, data, start, end):
-        checks = self.config['from_image']['md_keys']
-        none_values = self.config['from_image']['none_values']
-        service = self.config['from_image']['service']
-        size_sources = self.config['from_image']['size_keys']
+        checks = self.config['md_keys']
+        none_values = self.config['none_values']
+        service = self.config['service']
+        size_sources = self.config['size_keys']
 
         size = 0
         for entry in data:
@@ -183,3 +183,54 @@ class MagnumTransformer(BaseTransformer):
         hours = (end - start).total_seconds() / 3600.0
         return {name: max_vol * hours}
 
+
+class DatabaseUpTimeTransformer(UpTimeTransformer):
+    """
+    Transformer to calculate uptime based on states,
+    which is broken apart into flavor at point in time.
+    """
+
+    def _clean_entry(self, entry):
+        try:
+            timestamp = datetime.strptime(
+                entry['timestamp'], constants.date_format)
+        except ValueError:
+            timestamp = datetime.strptime(
+                entry['timestamp'], constants.date_format_f)
+
+        flavor = openstack.get_flavor_name(
+            entry['metadata'].get('flavor.id'))
+
+        result = {
+            'status': entry['metadata'].get('status'),
+            'flavor': flavor,
+            'timestamp': timestamp
+        }
+
+        return result
+
+
+class DatabaseManagementUpTimeTransformer(UpTimeTransformer):
+    """
+    Transformer to calculate uptime based on states,
+    which is broken apart into flavor at point in time.
+    """
+
+    def _clean_entry(self, entry):
+        management_service_name = self.config.get(
+            'service_name', 'd1.management')
+
+        try:
+            timestamp = datetime.strptime(
+                entry['timestamp'], constants.date_format)
+        except ValueError:
+            timestamp = datetime.strptime(
+                entry['timestamp'], constants.date_format_f)
+
+        result = {
+            'status': entry['metadata'].get('status'),
+            'flavor': management_service_name,
+            'timestamp': timestamp
+        }
+
+        return result

etc/meter_mappings.yml.sample

@@ -253,3 +253,48 @@
       sources:
         - name
 
+-
+  meter: database.instance
+  type: Database Instance
+  transformer: databaseuptime
+  unit: second
+  metadata:
+    name:
+      sources:
+        - name
+    datastore:
+      sources:
+        - datastore
+
+-
+  meter: database.instance
+  service: b1.standard
+  type: Database Volume
+  transformer: databasevolume
+  unit: gigabyte
+  res_id_template: "%s-volume"
+  metadata:
+    name:
+      sources:
+        - name
+      template: "%s - volume"
+    datastore:
+      sources:
+        - datastore
+
+-
+  meter: database.instance
+  type: Database Management
+  transformer: databasemanagementuptime
+  transformer_config:
+    service_name: d1.management
+  unit: second
+  res_id_template: "%s-management"
+  metadata:
+    name:
+      sources:
+        - name
+      template: "%s - Management Fee"
+    datastore:
+      sources:
+        - datastore

etc/transformer.yaml.sample

@@ -23,3 +23,9 @@ from_image:
   # where to get volume size from
   size_keys:
     - root_gb
+databaseuptime:
+  tracked_states:
+    ERROR
+databasemanagementuptime:
+  tracked_states:
+    ERROR

setup.cfg

@@ -46,6 +46,9 @@ distil.transformer =
     uptime = distil.transformer.conversion:UpTimeTransformer
     fromimage = distil.transformer.conversion:FromImageTransformer
     networkservice = distil.transformer.conversion:NetworkServiceTransformer
+    databaseuptime = distil.transformer.conversion:DatabaseUpTimeTransformer
+    databasevolume = distil.transformer.arithmetic:DatabaseVolumeMaxTransformer
+    databasemanagementuptime = distil.transformer.conversion:DatabaseManagementUpTimeTransformer
 
 distil.erp =
     odoo = distil.erp.drivers.odoo:OdooDriver