add transformers for trove

slightly alter how config is loaded to transformers.

Change-Id: I32bf34d270604e8c755160a7dac71e99a38f2a74

distil/collector/base.py

@@ -205,7 +205,9 @@ class BaseCollector(object):
         service = (mapping['service'] if 'service' in mapping
                    else mapping['meter'])
 
-        transformer = d_transformer.get_transformer(mapping['transformer'])
+        transformer = d_transformer.get_transformer(
+            mapping['transformer'],
+            override_config=mapping.get('transformer_config', {}))
 
         for res_id, entries in usage_by_resource.items():
             transformed = transformer.transform_usage(

distil/common/general.py

@@ -33,7 +33,7 @@ LOG = logging.getLogger(__name__)
 _TRANS_CONFIG = None
 
 
-def get_transformer_config():
+def get_transformer_config(name):
     global _TRANS_CONFIG
 
     if not _TRANS_CONFIG:
@@ -43,7 +43,7 @@ def get_transformer_config():
         except IOError as e:
             raise e
 
-    return _TRANS_CONFIG
+    return _TRANS_CONFIG.get(name, {})
 
 
 def get_windows(start, end):

distil/common/openstack.py

@@ -17,12 +17,14 @@ import re
 
 from ceilometerclient import client as ceilometerclient
 from cinderclient.v2 import client as cinderclient
+from cinderclient.exceptions import NotFound as CinderNotFound
 from glanceclient import client as glanceclient
 from keystoneauth1.identity import v3
 from keystoneauth1.exceptions import NotFound
 from keystoneauth1 import session
 from keystoneclient.v3 import client as ks_client
 from novaclient import client as novaclient
+from novaclient.exceptions import NotFound as NovaNotFound
 from oslo_config import cfg
 
 from distil.common import cache as distil_cache
@@ -30,7 +32,7 @@ from distil.common import general
 
 CONF = cfg.CONF
 KS_SESSION = None
-cache = defaultdict(list)
+cache = defaultdict(dict)
 ROOT_DEVICE_PATTERN = re.compile('^/dev/(x?v|s|h)da1?$')
 
 
@@ -163,20 +165,43 @@ def get_root_volume(instance_id):
 
 
 @general.disable_ssl_warnings
-def get_volume_type(volume_type):
+def get_flavor_name(flavor_id):
-    if not cache.get("volume_types"):
+    if flavor_id not in cache["flavors"]:
+        nova = get_nova_client()
+        try:
+            flavor_name = nova.flavors.get(flavor_id).name
+        except NovaNotFound:
+            return None
+        cache["flavors"][flavor_id] = flavor_name
+    return cache["flavors"][flavor_id]
+
+
+@general.disable_ssl_warnings
+def get_volume_type_for_volume(volume_id):
+    if volume_id not in cache["volume_id_to_type"]:
         cinder = get_cinder_client()
-        for vtype in cinder.volume_types.list():
+        try:
-            cache['volume_types'].append({'id': vtype.id, 'name': vtype.name})
+            vol = cinder.volumes.get(volume_id)
+        except CinderNotFound:
+            return None
+        cache["volume_id_to_type"][volume_id] = vol.volume_type
+    return cache["volume_id_to_type"][volume_id]
 
-    for vtype in cache['volume_types']:
-        # check name first, as that will be more common
-        if vtype['name'] == volume_type:
-            return volume_type
-        elif vtype['id'] == volume_type:
-            return vtype['name']
 
-    return None
+@general.disable_ssl_warnings
+def get_volume_type_name(volume_type):
+    if volume_type not in cache["volume_types"]:
+        cinder = get_cinder_client()
+        try:
+            vtype = cinder.volume_types.get(volume_type)
+        except CinderNotFound:
+            try:
+                vtype = cinder.volume_types.find(name=volume_type)
+            except CinderNotFound:
+                return None
+        cache["volume_types"][vtype.id] = vtype.name
+        cache["volume_types"][vtype.name] = vtype.name
+    return cache["volume_types"][volume_type]
 
 
 @general.disable_ssl_warnings

distil/tests/unit/transformer/test_arithmetic.py

@@ -20,7 +20,7 @@ from distil.common.constants import date_format
 from distil.common import general
 from distil.common import openstack
 from distil.tests.unit import base
-from distil.transformer import arithmetic
+from distil.transformer import get_transformer
 
 p = lambda t: datetime.datetime.strptime(t, date_format)
 
@@ -53,7 +53,7 @@ class TestMaxTransformer(base.DistilTestCase):
             {'timestamp': FAKE_DATA.t1.isoformat(), 'volume': 6},
         ]
 
-        xform = arithmetic.MaxTransformer()
+        xform = get_transformer('max')
         usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
                                       FAKE_DATA.t1)
 
@@ -71,7 +71,7 @@ class TestMaxTransformer(base.DistilTestCase):
             {'timestamp': FAKE_DATA.t1, 'volume': 25},
         ]
 
-        xform = arithmetic.MaxTransformer()
+        xform = get_transformer('max')
         usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
                                       FAKE_DATA.t1)
 
@@ -86,7 +86,7 @@ class TestMaxTransformer(base.DistilTestCase):
             {'timestamp': FAKE_DATA.t0, 'volume': None},
         ]
 
-        xform = arithmetic.MaxTransformer()
+        xform = get_transformer('max')
         usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
                                       FAKE_DATA.t1)
 
@@ -103,7 +103,7 @@ class TestMaxTransformer(base.DistilTestCase):
             {'timestamp': FAKE_DATA.t1, 'volume': 27},
         ]
 
-        xform = arithmetic.MaxTransformer()
+        xform = get_transformer('max')
         usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
                                       FAKE_DATA.t1)
 
@@ -135,7 +135,7 @@ class TestBlockStorageMaxTransformer(base.DistilTestCase):
              'metadata': {}},
         ]
 
-        xform = arithmetic.BlockStorageMaxTransformer()
+        xform = get_transformer('storagemax')
         usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
                                       FAKE_DATA.t1)
 
@@ -156,7 +156,7 @@ class TestBlockStorageMaxTransformer(base.DistilTestCase):
              'metadata': {}},
         ]
 
-        xform = arithmetic.BlockStorageMaxTransformer()
+        xform = get_transformer('storagemax')
         usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
                                       FAKE_DATA.t1)
 
@@ -172,7 +172,7 @@ class TestBlockStorageMaxTransformer(base.DistilTestCase):
              'metadata': {}},
         ]
 
-        xform = arithmetic.BlockStorageMaxTransformer()
+        xform = get_transformer('storagemax')
         usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
                                       FAKE_DATA.t1)
 
@@ -192,7 +192,7 @@ class TestBlockStorageMaxTransformer(base.DistilTestCase):
              'metadata': {}},
         ]
 
-        xform = arithmetic.BlockStorageMaxTransformer()
+        xform = get_transformer('storagemax')
         usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
                                       FAKE_DATA.t1)
 
@@ -241,7 +241,7 @@ class TestObjectStorageMaxTransformer(base.DistilTestCase):
              'metadata': {}},
         ]
 
-        xform = arithmetic.ObjectStorageMaxTransformer()
+        xform = get_transformer('objectstoragemax')
         usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
                                       FAKE_DATA.t1)
 
@@ -270,7 +270,7 @@ class TestObjectStorageMaxTransformer(base.DistilTestCase):
              'metadata': {}},
         ]
 
-        xform = arithmetic.ObjectStorageMaxTransformer()
+        xform = get_transformer('objectstoragemax')
         usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
                                       FAKE_DATA.t1)
 
@@ -290,7 +290,7 @@ class TestObjectStorageMaxTransformer(base.DistilTestCase):
              'metadata': {}},
         ]
 
-        xform = arithmetic.ObjectStorageMaxTransformer()
+        xform = get_transformer('objectstoragemax')
         usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
                                       FAKE_DATA.t1)
 
@@ -318,7 +318,7 @@ class TestObjectStorageMaxTransformer(base.DistilTestCase):
              'metadata': {}},
         ]
 
-        xform = arithmetic.ObjectStorageMaxTransformer()
+        xform = get_transformer('objectstoragemax')
         usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
                                       FAKE_DATA.t1)
 
@@ -339,7 +339,7 @@ class TestSumTransformer(base.DistilTestCase):
             {'timestamp': '2014-01-01T01:00:00', 'volume': 1},
         ]
 
-        xform = arithmetic.SumTransformer()
+        xform = get_transformer('sum')
         usage = xform.transform_usage('fake_meter', data, FAKE_DATA.t0,
                                       FAKE_DATA.t1)
 
@@ -354,7 +354,7 @@ class TestSumTransformer(base.DistilTestCase):
             {'timestamp': FAKE_DATA.t0.isoformat(), 'volume': None},
         ]
 
-        xform = arithmetic.SumTransformer()
+        xform = get_transformer('sum')
         usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
                                       FAKE_DATA.t1)
 
@@ -371,8 +371,58 @@ class TestSumTransformer(base.DistilTestCase):
             {'timestamp': FAKE_DATA.t0_50.isoformat(), 'volume': 25},
         ]
 
-        xform = arithmetic.SumTransformer()
+        xform = get_transformer('sum')
         usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
                                       FAKE_DATA.t1)
 
         self.assertEqual({'some_meter': 50}, usage)
+
+
+@mock.patch.object(general, 'get_transformer_config', mock.Mock())
+class TestDatabaseVolumeMaxTransformer(base.DistilTestCase):
+
+    @mock.patch.object(
+        openstack, 'get_volume_type_for_volume',
+        mock.Mock(return_value='b1.nvme1000'))
+    def test_all_different_values(self):
+        """
+        Tests that the transformer correctly grabs the highest value,
+        when all values are different.
+        """
+
+        data = [
+            {'timestamp': FAKE_DATA.t0, 'volume': 1,
+             'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
+             'project_id': '55d37509be3142de963caf82a9c7c447',
+             'metadata': {'volume.size': '24', 'volume_id': 'vol_id'}},
+            {'timestamp': FAKE_DATA.t0_10, 'volume': 1,
+             'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
+             'project_id': '55d37509be3142de963caf82a9c7c447',
+             'metadata': {'volume.size': '13', 'volume_id': 'vol_id'}},
+            {'timestamp': FAKE_DATA.t0_20, 'volume': 1,
+             'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
+             'project_id': '55d37509be3142de963caf82a9c7c447',
+             'metadata': {'volume.size': '7', 'volume_id': 'vol_id'}},
+            {'timestamp': FAKE_DATA.t0_30, 'volume': 1,
+             'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
+             'project_id': '55d37509be3142de963caf82a9c7c447',
+             'metadata': {'volume.size': '13', 'volume_id': 'vol_id'}},
+            {'timestamp': FAKE_DATA.t0_40, 'volume': 1,
+             'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
+             'project_id': '55d37509be3142de963caf82a9c7c447',
+             'metadata': {'volume.size': '3', 'volume_id': 'vol_id'}},
+            {'timestamp': FAKE_DATA.t0_50, 'volume': 1,
+             'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
+             'project_id': '55d37509be3142de963caf82a9c7c447',
+             'metadata': {'volume.size': '25', 'volume_id': 'vol_id'}},
+            {'timestamp': FAKE_DATA.t1, 'volume': 1,
+             'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
+             'project_id': '55d37509be3142de963caf82a9c7c447',
+             'metadata': {'volume.size': '13', 'volume_id': 'vol_id'}},
+        ]
+
+        xform = get_transformer('databasevolumemax')
+        usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
+                                      FAKE_DATA.t1)
+
+        self.assertEqual({'b1.nvme1000': 25}, usage)

distil/tests/unit/transformer/test_conversion.py

@@ -17,8 +17,9 @@ import mock
 
 from distil.common.constants import date_format
 from distil.common import general
+from distil.common import openstack
 from distil.tests.unit import base
-from distil.transformer import conversion
+from distil.transformer import get_transformer
 
 p = lambda t: datetime.datetime.strptime(t, date_format)
 
@@ -41,17 +42,28 @@ FAKE_CONFIG = {
         "tracked_states": ["active", "building", "paused", "rescued",
                            "resized"]
     },
-    "from_image": {
+    "fromimage": {
         "service": "volume.size",
         "md_keys": ["image_ref", "image_meta.base_image_ref"],
         "none_values": ["None", ""],
         "size_keys": ["root_gb"]
+    },
+    "databaseuptime": {
+        "tracked_states": ["ACTIVE", "BUILD"]
+    },
+    "databasemanagementuptime": {
+        "tracked_states": ["ACTIVE", "BUILD"],
+        "service_name": "d1.managment"
     }
 }
 
 
+def fake_get_transformer_config(name):
+    return FAKE_CONFIG.get(name, {})
+
+
 @mock.patch.object(general, 'get_transformer_config',
-                   mock.Mock(return_value=FAKE_CONFIG))
+                   fake_get_transformer_config)
 class TestUpTimeTransformer(base.DistilTestCase):
     def test_trivial_run(self):
         """
@@ -59,7 +71,7 @@ class TestUpTimeTransformer(base.DistilTestCase):
         """
         state = []
 
-        xform = conversion.UpTimeTransformer()
+        xform = get_transformer('uptime')
         result = xform.transform_usage('state', state, FAKE_DATA.t0,
                                        FAKE_DATA.t1)
 
@@ -79,7 +91,7 @@ class TestUpTimeTransformer(base.DistilTestCase):
                           'status': 'active'}}
         ]
 
-        xform = conversion.UpTimeTransformer()
+        xform = get_transformer('uptime')
         result = xform.transform_usage('state', state, FAKE_DATA.t0,
                                        FAKE_DATA.t1)
 
@@ -100,7 +112,7 @@ class TestUpTimeTransformer(base.DistilTestCase):
                           'status': 'stopped'}}
         ]
 
-        xform = conversion.UpTimeTransformer()
+        xform = get_transformer('uptime')
         result = xform.transform_usage('state', state, FAKE_DATA.t0,
                                        FAKE_DATA.t1)
 
@@ -122,7 +134,7 @@ class TestUpTimeTransformer(base.DistilTestCase):
                           'status': 'stopped'}}
         ]
 
-        xform = conversion.UpTimeTransformer()
+        xform = get_transformer('uptime')
         result = xform.transform_usage('state', state, FAKE_DATA.t0,
                                        FAKE_DATA.t1)
 
@@ -145,7 +157,7 @@ class TestUpTimeTransformer(base.DistilTestCase):
                           'status': 'active'}}
         ]
 
-        xform = conversion.UpTimeTransformer()
+        xform = get_transformer('uptime')
         result = xform.transform_usage('state', state, FAKE_DATA.t0,
                                        FAKE_DATA.t1)
 
@@ -168,7 +180,7 @@ class TestUpTimeTransformer(base.DistilTestCase):
                           'status': 'active'}}
         ]
 
-        xform = conversion.UpTimeTransformer()
+        xform = get_transformer('uptime')
         result = xform.transform_usage('state', state, FAKE_DATA.t0,
                                        FAKE_DATA.t1)
 
@@ -192,7 +204,7 @@ class TestUpTimeTransformer(base.DistilTestCase):
                           'status': 'active'}}
         ]
 
-        xform = conversion.UpTimeTransformer()
+        xform = get_transformer('uptime')
         result = xform.transform_usage('state', state, FAKE_DATA.t0,
                                        FAKE_DATA.t1)
 
@@ -212,7 +224,7 @@ class TestUpTimeTransformer(base.DistilTestCase):
                           'state': 'active'}}
         ]
 
-        xform = conversion.UpTimeTransformer()
+        xform = get_transformer('uptime')
         result = xform.transform_usage('state', state, FAKE_DATA.t0,
                                        FAKE_DATA.t1)
 
@@ -230,7 +242,7 @@ class TestUpTimeTransformer(base.DistilTestCase):
              'metadata': {'instance_type': FAKE_DATA.flavor}}
         ]
 
-        xform = conversion.UpTimeTransformer()
+        xform = get_transformer('uptime')
         result = xform.transform_usage('state', state, FAKE_DATA.t0,
                                        FAKE_DATA.t1)
 
@@ -252,7 +264,7 @@ class TestUpTimeTransformer(base.DistilTestCase):
                           'status': 'active'}}
         ]
 
-        xform = conversion.UpTimeTransformer()
+        xform = get_transformer('uptime')
         result = xform.transform_usage(
             'instance',
             entries,
@@ -265,7 +277,7 @@ class TestUpTimeTransformer(base.DistilTestCase):
 
 
 @mock.patch.object(general, 'get_transformer_config',
-                   mock.Mock(return_value=FAKE_CONFIG))
+                   fake_get_transformer_config)
 class TestFromImageTransformer(base.DistilTestCase):
     """
     These tests rely on config settings for from_image,
@@ -290,7 +302,7 @@ class TestFromImageTransformer(base.DistilTestCase):
              'metadata': {'image_ref': "None"}}
         ]
 
-        xform = conversion.FromImageTransformer()
+        xform = get_transformer('fromimage')
 
         usage = xform.transform_usage('instance', data, FAKE_DATA.t0,
                                       FAKE_DATA.t1)
@@ -313,7 +325,7 @@ class TestFromImageTransformer(base.DistilTestCase):
              'metadata': {'image_ref': "None"}}
         ]
 
-        xform = conversion.FromImageTransformer()
+        xform = get_transformer('fromimage')
         usage = xform.transform_usage('instance', data, FAKE_DATA.t0,
                                       FAKE_DATA.t1)
 
@@ -335,7 +347,7 @@ class TestFromImageTransformer(base.DistilTestCase):
                           'root_gb': "20"}}
         ]
 
-        xform = conversion.FromImageTransformer()
+        xform = get_transformer('fromimage')
         usage = xform.transform_usage('instance', data, FAKE_DATA.t0,
                                       FAKE_DATA.t1)
 
@@ -358,7 +370,7 @@ class TestFromImageTransformer(base.DistilTestCase):
                           'root_gb': "20"}}
         ]
 
-        xform = conversion.FromImageTransformer()
+        xform = get_transformer('fromimage')
         usage = xform.transform_usage('instance', data, FAKE_DATA.t0,
                                       FAKE_DATA.t1)
 
@@ -366,7 +378,7 @@ class TestFromImageTransformer(base.DistilTestCase):
 
 
 @mock.patch.object(general, 'get_transformer_config',
-                   mock.Mock(return_value=FAKE_CONFIG))
+                   fake_get_transformer_config)
 class TestNetworkServiceTransformer(base.DistilTestCase):
     def test_basic_sum(self):
         """Tests that the transformer correctly calculate the sum value.
@@ -378,7 +390,7 @@ class TestNetworkServiceTransformer(base.DistilTestCase):
             {'timestamp': '2014-01-01T01:00:00', 'volume': 2},
         ]
 
-        xform = conversion.NetworkServiceTransformer()
+        xform = get_transformer('networkservice')
         usage = xform.transform_usage('fake_meter', data, FAKE_DATA.t0,
                                       FAKE_DATA.t1)
 
@@ -394,7 +406,7 @@ class TestNetworkServiceTransformer(base.DistilTestCase):
             {'timestamp': '2014-01-01T01:00:00', 'volume': 2},
         ]
 
-        xform = conversion.NetworkServiceTransformer()
+        xform = get_transformer('networkservice')
         usage = xform.transform_usage('fake_meter', data, FAKE_DATA.t0,
                                       FAKE_DATA.t1)
 
@@ -413,7 +425,7 @@ class TestMagnumTransformer(base.DistilTestCase):
             {'timestamp': '2014-01-01T01:00:00', 'volume': 2},
         ]
 
-        xform = conversion.MagnumTransformer()
+        xform = get_transformer('magnum')
         usage = xform.transform_usage('fake_meter', data, FAKE_DATA.t0,
                                       FAKE_DATA.t1)
 
@@ -429,8 +441,213 @@ class TestMagnumTransformer(base.DistilTestCase):
             {'timestamp': '2014-01-01T01:00:00', 'volume': 18},
         ]
 
-        xform = conversion.MagnumTransformer()
+        xform = get_transformer('magnum')
         usage = xform.transform_usage('fake_meter', data, FAKE_DATA.t0,
                                       FAKE_DATA.t1)
 
         self.assertEqual({'fake_meter': 0}, usage)
+
+
+@mock.patch.object(general, 'get_transformer_config',
+                   fake_get_transformer_config)
+class TestDatabaseUpTimeTransformer(base.DistilTestCase):
+
+    @mock.patch.object(
+        openstack, 'get_flavor_name',
+        mock.Mock(return_value=FAKE_DATA.flavor))
+    def test_online_constant_flavor(self):
+        """
+        Test that a machine online for a 1h period with constant
+        flavor works and gives 1h of uptime.
+        """
+        state = [
+            {'timestamp': FAKE_DATA.t0.isoformat(),
+             'metadata': {'flavor.id': FAKE_DATA.flavor,
+                          'status': 'ACTIVE'}},
+            {'timestamp': FAKE_DATA.t1.isoformat(),
+             'metadata': {'flavor.id': FAKE_DATA.flavor,
+                          'status': 'ACTIVE'}}
+        ]
+
+        xform = get_transformer('databaseuptime')
+        result = xform.transform_usage('state', state, FAKE_DATA.t0,
+                                       FAKE_DATA.t1)
+
+        self.assertEqual({FAKE_DATA.flavor: 3600}, result)
+
+    @mock.patch.object(
+        openstack, 'get_flavor_name',
+        mock.Mock(return_value=FAKE_DATA.flavor))
+    def test_offline_constant_flavor(self):
+        """
+        Test that a machine offline for a 1h period with constant flavor
+        works and gives zero uptime.
+        """
+
+        state = [
+            {'timestamp': FAKE_DATA.t0.isoformat(),
+             'metadata': {'flavor.id': FAKE_DATA.flavor,
+                          'status': 'stopped'}},
+            {'timestamp': FAKE_DATA.t1.isoformat(),
+             'metadata': {'flavor.id': FAKE_DATA.flavor,
+                          'status': 'stopped'}}
+        ]
+
+        xform = get_transformer('databaseuptime')
+        result = xform.transform_usage('state', state, FAKE_DATA.t0,
+                                       FAKE_DATA.t1)
+
+        self.assertEqual({}, result)
+
+    @mock.patch.object(
+        openstack, 'get_flavor_name',
+        mock.Mock(return_value=FAKE_DATA.flavor))
+    def test_shutdown_during_period(self):
+        """
+        Test that a machine run for 0.5 then shutdown gives 0.5h uptime.
+        """
+        state = [
+            {'timestamp': FAKE_DATA.t0.isoformat(),
+             'metadata': {'flavor.id': FAKE_DATA.flavor,
+                          'status': 'ACTIVE'}},
+            {'timestamp': FAKE_DATA.t0_30.isoformat(),
+             'metadata': {'flavor.id': FAKE_DATA.flavor,
+                          'status': 'stopped'}},
+            {'timestamp': FAKE_DATA.t1.isoformat(),
+             'metadata': {'flavor.id': FAKE_DATA.flavor,
+                          'status': 'stopped'}}
+        ]
+
+        xform = get_transformer('databaseuptime')
+        result = xform.transform_usage('state', state, FAKE_DATA.t0,
+                                       FAKE_DATA.t1)
+
+        self.assertEqual({FAKE_DATA.flavor: 1800}, result)
+
+    def test_online_flavor_change(self):
+        """
+        Test that a machine run for 0.5h as m1.tiny, resized to m1.large,
+        and run for a further 0.5 yields 0.5h of uptime in each class.
+        """
+        state = [
+            {'timestamp': FAKE_DATA.t0.isoformat(),
+             'metadata': {'flavor.id': FAKE_DATA.flavor,
+                          'status': 'ACTIVE'}},
+            {'timestamp': FAKE_DATA.t0_30.isoformat(),
+             'metadata': {'flavor.id': FAKE_DATA.flavor2,
+                          'status': 'ACTIVE'}},
+            {'timestamp': FAKE_DATA.t1.isoformat(),
+             'metadata': {'flavor.id': FAKE_DATA.flavor2,
+                          'status': 'ACTIVE'}}
+        ]
+
+        xform = get_transformer('databaseuptime')
+
+        def fake_get_flavor(name):
+            return name
+
+        with mock.patch.object(
+                openstack, 'get_flavor_name', fake_get_flavor):
+            result = xform.transform_usage(
+                'state', state, FAKE_DATA.t0, FAKE_DATA.t1)
+
+        self.assertDictEqual(
+            {FAKE_DATA.flavor: 1800, FAKE_DATA.flavor2: 1800},
+            result
+        )
+
+    @mock.patch.object(
+        openstack, 'get_flavor_name',
+        mock.Mock(return_value=FAKE_DATA.flavor))
+    def test_no_state_in_metedata(self):
+        """
+        Test that the transformer doesn't fall over if there isn't one of
+        the two state/status key options in the metadata.
+        """
+        state = [
+            {'timestamp': FAKE_DATA.t0.isoformat(),
+             'metadata': {'flavor.id': FAKE_DATA.flavor}},
+            {'timestamp': FAKE_DATA.t1.isoformat(),
+             'metadata': {'flavor.id': FAKE_DATA.flavor}}
+        ]
+
+        xform = get_transformer('databaseuptime')
+        result = xform.transform_usage('state', state, FAKE_DATA.t0,
+                                       FAKE_DATA.t1)
+
+        self.assertEqual({}, result)
+
+
+@mock.patch.object(general, 'get_transformer_config',
+                   fake_get_transformer_config)
+class TestDatabaseManagementUpTimeTransformer(base.DistilTestCase):
+
+    @mock.patch.object(
+        openstack, 'get_flavor_name',
+        mock.Mock(return_value=FAKE_DATA.flavor))
+    def test_online_constant_flavor(self):
+        """
+        Test that a machine online for a 1h period with constant
+        flavor works and gives 1h of uptime.
+        """
+        state = [
+            {'timestamp': FAKE_DATA.t0.isoformat(),
+             'metadata': {'status': 'ACTIVE'}},
+            {'timestamp': FAKE_DATA.t1.isoformat(),
+             'metadata': {'status': 'ACTIVE'}}
+        ]
+
+        xform = get_transformer('databasemanagementuptime')
+        result = xform.transform_usage('state', state, FAKE_DATA.t0,
+                                       FAKE_DATA.t1)
+
+        self.assertEqual({"d1.managment": 3600}, result)
+
+    @mock.patch.object(
+        openstack, 'get_flavor_name',
+        mock.Mock(return_value=FAKE_DATA.flavor))
+    def test_offline_constant_flavor(self):
+        """
+        Test that a machine offline for a 1h period with constant flavor
+        works and gives zero uptime.
+        """
+
+        state = [
+            {'timestamp': FAKE_DATA.t0.isoformat(),
+             'metadata': {'flavor.id': FAKE_DATA.flavor,
+                          'status': 'stopped'}},
+            {'timestamp': FAKE_DATA.t1.isoformat(),
+             'metadata': {'flavor.id': FAKE_DATA.flavor,
+                          'status': 'stopped'}}
+        ]
+
+        xform = get_transformer('databasemanagementuptime')
+        result = xform.transform_usage('state', state, FAKE_DATA.t0,
+                                       FAKE_DATA.t1)
+
+        self.assertEqual({}, result)
+
+    @mock.patch.object(
+        openstack, 'get_flavor_name',
+        mock.Mock(return_value=FAKE_DATA.flavor))
+    def test_shutdown_during_period(self):
+        """
+        Test that a machine run for 0.5 then shutdown gives 0.5h uptime.
+        """
+        state = [
+            {'timestamp': FAKE_DATA.t0.isoformat(),
+             'metadata': {'flavor.id': FAKE_DATA.flavor,
+                          'status': 'ACTIVE'}},
+            {'timestamp': FAKE_DATA.t0_30.isoformat(),
+             'metadata': {'flavor.id': FAKE_DATA.flavor,
+                          'status': 'stopped'}},
+            {'timestamp': FAKE_DATA.t1.isoformat(),
+             'metadata': {'flavor.id': FAKE_DATA.flavor,
+                          'status': 'stopped'}}
+        ]
+
+        xform = get_transformer('databasemanagementuptime')
+        result = xform.transform_usage('state', state, FAKE_DATA.t0,
+                                       FAKE_DATA.t1)
+
+        self.assertEqual({"d1.managment": 1800}, result)

distil/transformer/__init__.py

@@ -20,8 +20,10 @@ from distil.common import general
 
 class BaseTransformer(object):
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, name, override_config=None):
-        self.config = general.get_transformer_config()
+        self.config = general.get_transformer_config(name)
+        if override_config:
+            self.config.update(override_config)
 
     def transform_usage(self, meter_name, raw_data, start_at, end_at):
         return self._transform_usage(meter_name, raw_data, start_at, end_at)
@@ -35,5 +37,6 @@ def get_transformer(name, **kwargs):
         'distil.transformer',
         name,
         invoke_on_load=True,
+        invoke_args=(name,),
         invoke_kwds=kwargs
     ).driver

distil/transformer/arithmetic.py

@@ -59,7 +59,7 @@ class BlockStorageMaxTransformer(MaxTransformer):
 
         if "volume_type" in data[-1]['metadata']:
             vtype = data[-1]['metadata']['volume_type']
-            service = openstack.get_volume_type(vtype)
+            service = openstack.get_volume_type_name(vtype)
             if not service:
                 service = name
         else:
@@ -69,6 +69,30 @@ class BlockStorageMaxTransformer(MaxTransformer):
         return {service: max_vol * hours}
 
 
+class DatabaseVolumeMaxTransformer(BaseTransformer):
+    """
+    Variantion on the GaugeMax Transformer that checks for
+    volume_type and uses that as the service, or uses the
+    default service name.
+
+    It also gets the actual volume size from metadata.
+    """
+
+    def _transform_usage(self, name, data, start, end):
+        if not data:
+            return None
+
+        max_vol = max([int(v["metadata"]["volume.size"]) for v in data])
+
+        volume_type = openstack.get_volume_type_for_volume(
+            data[-1]['metadata']['volume_id'])
+        if not volume_type:
+            return None
+
+        hours = (end - start).total_seconds() / 3600.0
+        return {volume_type: max_vol * hours}
+
+
 class ObjectStorageMaxTransformer(MaxTransformer):
     """
     Variantion on the GaugeMax Transformer that checks for

distil/transformer/conversion.py

@@ -40,7 +40,7 @@ class UpTimeTransformer(BaseTransformer):
 
     def _transform_usage(self, name, data, start, end):
         # get tracked states from config
-        tracked = self.config['uptime']['tracked_states']
+        tracked = self.config['tracked_states']
 
         usage_dict = {}
 
@@ -118,10 +118,10 @@ class FromImageTransformer(BaseTransformer):
     """
 
     def _transform_usage(self, name, data, start, end):
-        checks = self.config['from_image']['md_keys']
+        checks = self.config['md_keys']
-        none_values = self.config['from_image']['none_values']
+        none_values = self.config['none_values']
-        service = self.config['from_image']['service']
+        service = self.config['service']
-        size_sources = self.config['from_image']['size_keys']
+        size_sources = self.config['size_keys']
 
         size = 0
         for entry in data:
@@ -183,3 +183,54 @@ class MagnumTransformer(BaseTransformer):
         hours = (end - start).total_seconds() / 3600.0
         return {name: max_vol * hours}
 
+
+class DatabaseUpTimeTransformer(UpTimeTransformer):
+    """
+    Transformer to calculate uptime based on states,
+    which is broken apart into flavor at point in time.
+    """
+
+    def _clean_entry(self, entry):
+        try:
+            timestamp = datetime.strptime(
+                entry['timestamp'], constants.date_format)
+        except ValueError:
+            timestamp = datetime.strptime(
+                entry['timestamp'], constants.date_format_f)
+
+        flavor = openstack.get_flavor_name(
+            entry['metadata'].get('flavor.id'))
+
+        result = {
+            'status': entry['metadata'].get('status'),
+            'flavor': flavor,
+            'timestamp': timestamp
+        }
+
+        return result
+
+
+class DatabaseManagementUpTimeTransformer(UpTimeTransformer):
+    """
+    Transformer to calculate uptime based on states,
+    which is broken apart into flavor at point in time.
+    """
+
+    def _clean_entry(self, entry):
+        management_service_name = self.config.get(
+            'service_name', 'd1.management')
+
+        try:
+            timestamp = datetime.strptime(
+                entry['timestamp'], constants.date_format)
+        except ValueError:
+            timestamp = datetime.strptime(
+                entry['timestamp'], constants.date_format_f)
+
+        result = {
+            'status': entry['metadata'].get('status'),
+            'flavor': management_service_name,
+            'timestamp': timestamp
+        }
+
+        return result

etc/meter_mappings.yml.sample

@@ -253,3 +253,48 @@
       sources:
         - name
 
+-
+  meter: database.instance
+  type: Database Instance
+  transformer: databaseuptime
+  unit: second
+  metadata:
+    name:
+      sources:
+        - name
+    datastore:
+      sources:
+        - datastore
+
+-
+  meter: database.instance
+  service: b1.standard
+  type: Database Volume
+  transformer: databasevolumemax
+  unit: gigabyte
+  res_id_template: "%s-volume"
+  metadata:
+    name:
+      sources:
+        - name
+      template: "%s - volume"
+    datastore:
+      sources:
+        - datastore
+
+-
+  meter: database.instance
+  type: Database Management
+  transformer: databasemanagementuptime
+  transformer_config:
+    service_name: d1.management
+  unit: second
+  res_id_template: "%s-management"
+  metadata:
+    name:
+      sources:
+        - name
+      template: "%s - Management Fee"
+    datastore:
+      sources:
+        - datastore

etc/transformer.yaml.sample

@@ -11,7 +11,7 @@ uptime:
     - suspended
     - shutoff
     - stopped
-from_image:
+fromimage:
   service: b1.standard
   # What metadata values to check
   md_keys:
@@ -23,3 +23,30 @@ from_image:
   # where to get volume size from
   size_keys:
     - root_gb
+databaseuptime:
+  tracked_states:
+    - HEALTHY
+    - ACTIVE
+    - BLOCKED
+    - REBOOT
+    - RESIZE
+    - BACKUP
+    - SHUTDOWN
+    - RESTART_REQUIRED
+    - PROMOTE
+    - EJECT
+    - UPGRADE
+    - DETACH
+databasemanagementuptime:
+    - HEALTHY
+    - ACTIVE
+    - BLOCKED
+    - REBOOT
+    - RESIZE
+    - BACKUP
+    - SHUTDOWN
+    - RESTART_REQUIRED
+    - PROMOTE
+    - EJECT
+    - UPGRADE
+    - DETACH

setup.cfg

@@ -46,6 +46,10 @@ distil.transformer =
     uptime = distil.transformer.conversion:UpTimeTransformer
     fromimage = distil.transformer.conversion:FromImageTransformer
     networkservice = distil.transformer.conversion:NetworkServiceTransformer
+    magnum = distil.transformer.conversion:MagnumTransformer
+    databaseuptime = distil.transformer.conversion:DatabaseUpTimeTransformer
+    databasevolumemax = distil.transformer.arithmetic:DatabaseVolumeMaxTransformer
+    databasemanagementuptime = distil.transformer.conversion:DatabaseManagementUpTimeTransformer
 
 distil.erp =
     odoo = distil.erp.drivers.odoo:OdooDriver

tox.ini

@@ -1,11 +1,11 @@
 [tox]
-envlist = py35,py27,pep8
+envlist = py3,py27,pep8
 minversion = 1.6
 skipsdist = True
 
 [testenv]
 usedevelop = True
-install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/master/upper-constraints.txt} {opts} {packages}
+install_command = pip install -c https://opendev.org/openstack/requirements/raw/branch/stable/ussuri/upper-constraints.txt {opts} {packages}
 setenv =
     VIRTUAL_ENV={envdir}
     DISTIL_TESTS_CONFIGS_DIR={toxinidir}/distil/tests/etc/