Add support of metering volume related resources

This change add the support of metering size of volume/snapshot/backup.

Blueprint add-support-of-metering-volume-related-resources
Change-Id: I9c6ec1e79c267255e7839d70324726e56bc6d768

ceilometer/opts.py

@@ -51,6 +51,7 @@ import ceilometer.sample
 import ceilometer.service
 import ceilometer.storage
 import ceilometer.utils
+import ceilometer.volume.discovery
 
 
 def list_opts():
@@ -107,7 +108,8 @@ def list_opts():
                          ceilometer.neutron_client.SERVICE_OPTS,
                          ceilometer.nova_client.SERVICE_OPTS,
                          ceilometer.objectstore.rgw.SERVICE_OPTS,
-                         ceilometer.objectstore.swift.SERVICE_OPTS,)),
+                         ceilometer.objectstore.swift.SERVICE_OPTS,
+                         ceilometer.volume.discovery.SERVICE_OPTS,)),
         ('storage', ceilometer.dispatcher.STORAGE_OPTS),
         ('vmware', ceilometer.compute.virt.vmware.inspector.OPTS),
         ('xenapi', ceilometer.compute.virt.xenapi.inspector.OPTS),

ceilometer/tests/unit/volume/test_cinder.py

@@ -0,0 +1,162 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import mock
+from oslo_config import fixture as fixture_config
+
+from ceilometer.agent import manager
+import ceilometer.tests.base as base
+from ceilometer.volume import cinder
+
+VOLUME_LIST = [
+    type('Volume', (object,),
+         {u'migration_status': None,
+          u'attachments': [
+              {u'server_id': u'1ae69721-d071-4156-a2bd-b11bb43ec2e3',
+               u'attachment_id': u'f903d95e-f999-4a34-8be7-119eadd9bb4f',
+               u'attached_at': u'2016-07-14T03:55:57.000000',
+               u'host_name': None,
+               u'volume_id': u'd94c18fb-b680-4912-9741-da69ee83c94f',
+               u'device': u'/dev/vdb',
+               u'id': u'd94c18fb-b680-4912-9741-da69ee83c94f'}],
+          u'links': [{
+              u'href': u'http://fake_link3',
+              u'rel': u'self'},
+              {
+                  u'href': u'http://fake_link4',
+                  u'rel': u'bookmark'}],
+          u'availability_zone': u'nova',
+          u'os-vol-host-attr:host': u'test@lvmdriver-1#lvmdriver-1',
+          u'encrypted': False,
+          u'updated_at': u'2016-07-14T03:55:57.000000',
+          u'replication_status': u'disabled',
+          u'snapshot_id': None,
+          u'id': u'd94c18fb-b680-4912-9741-da69ee83c94f',
+          u'size': 1,
+          u'user_id': u'be255bd31eb944578000fc762fde6dcf',
+          u'os-vol-tenant-attr:tenant_id': u'6824974c08974d4db864bbaa6bc08303',
+          u'os-vol-mig-status-attr:migstat': None,
+          u'metadata': {u'readonly': u'False', u'attached_mode': u'rw'},
+          u'status': u'in-use',
+          u'description': None,
+          u'multiattach': False,
+          u'source_volid': None,
+          u'consistencygroup_id': None,
+          u'os-vol-mig-status-attr:name_id': None,
+          u'name': None,
+          u'bootable': u'false',
+          u'created_at': u'2016-06-23T08:27:45.000000',
+          u'volume_type': u'lvmdriver-1'})
+]
+
+SNAPSHOT_LIST = [
+    type('VolumeSnapshot', (object,),
+         {u'status': u'available',
+          u'os-extended-snapshot-attributes:progress': u'100%',
+          u'description': None,
+          u'os-extended-snapshot-attributes:project_id':
+              u'6824974c08974d4db864bbaa6bc08303',
+          u'size': 1,
+          u'updated_at': u'2016-10-19T07:56:55.000000',
+          u'id': u'b1ea6783-f952-491e-a4ed-23a6a562e1cf',
+          u'volume_id': u'6f27bc42-c834-49ea-ae75-8d1073b37806',
+          u'metadata': {},
+          u'created_at': u'2016-10-19T07:56:55.000000',
+          u'name': None})
+]
+
+BACKUP_LIST = [
+    type('VolumeBackup', (object,),
+         {u'status': u'available',
+          u'object_count': 0,
+          u'container': None,
+          u'name': None,
+          u'links': [{
+              u'href': u'http://fake_urla',
+              u'rel': u'self'}, {
+              u'href': u'http://fake_urlb',
+              u'rel': u'bookmark'}],
+          u'availability_zone': u'nova',
+          u'created_at': u'2016-10-19T06:55:23.000000',
+          u'snapshot_id': None,
+          u'updated_at': u'2016-10-19T06:55:23.000000',
+          u'data_timestamp': u'2016-10-19T06:55:23.000000',
+          u'description': None,
+          u'has_dependent_backups': False,
+          u'volume_id': u'6f27bc42-c834-49ea-ae75-8d1073b37806',
+          u'fail_reason': u"",
+          u'is_incremental': False,
+          u'id': u'75a52125-85ff-4a8d-b2aa-580f3b22273f',
+          u'size': 1})
+]
+
+
+class TestVolumeSizePollster(base.BaseTestCase):
+    @mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
+    def setUp(self):
+        super(TestVolumeSizePollster, self).setUp()
+        self.CONF = self.useFixture(fixture_config.Config()).conf
+        self.manager = manager.AgentManager(0, self.CONF)
+        self.pollster = cinder.VolumeSizePollster(self.CONF)
+
+    def test_volume_size_pollster(self):
+        volume_size_samples = list(
+            self.pollster.get_samples(self.manager, {}, resources=VOLUME_LIST))
+        self.assertEqual(1, len(volume_size_samples))
+        self.assertEqual('volume.size', volume_size_samples[0].name)
+        self.assertEqual(1, volume_size_samples[0].volume)
+        self.assertEqual('6824974c08974d4db864bbaa6bc08303',
+                         volume_size_samples[0].project_id)
+        self.assertEqual('d94c18fb-b680-4912-9741-da69ee83c94f',
+                         volume_size_samples[0].resource_id)
+
+
+class TestVolumeSnapshotSizePollster(base.BaseTestCase):
+    @mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
+    def setUp(self):
+        super(TestVolumeSnapshotSizePollster, self).setUp()
+        self.CONF = self.useFixture(fixture_config.Config()).conf
+        self.manager = manager.AgentManager(0, self.CONF)
+        self.pollster = cinder.VolumeSnapshotSize(self.CONF)
+
+    def test_volume_snapshot_size_pollster(self):
+        volume_snapshot_size_samples = list(
+            self.pollster.get_samples(
+                self.manager, {}, resources=SNAPSHOT_LIST))
+        self.assertEqual(1, len(volume_snapshot_size_samples))
+        self.assertEqual('volume.snapshot.size',
+                         volume_snapshot_size_samples[0].name)
+        self.assertEqual(1, volume_snapshot_size_samples[0].volume)
+        self.assertEqual('6824974c08974d4db864bbaa6bc08303',
+                         volume_snapshot_size_samples[0].project_id)
+        self.assertEqual('b1ea6783-f952-491e-a4ed-23a6a562e1cf',
+                         volume_snapshot_size_samples[0].resource_id)
+
+
+class TestVolumeBackupSizePollster(base.BaseTestCase):
+    @mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
+    def setUp(self):
+        super(TestVolumeBackupSizePollster, self).setUp()
+        self.CONF = self.useFixture(fixture_config.Config()).conf
+        self.manager = manager.AgentManager(0, self.CONF)
+        self.pollster = cinder.VolumeBackupSize(self.CONF)
+
+    def test_volume_backup_size_pollster(self):
+        volume_backup_size_samples = list(
+            self.pollster.get_samples(self.manager, {}, resources=BACKUP_LIST))
+        self.assertEqual(1, len(volume_backup_size_samples))
+        self.assertEqual('volume.backup.size',
+                         volume_backup_size_samples[0].name)
+        self.assertEqual(1, volume_backup_size_samples[0].volume)
+        self.assertEqual('75a52125-85ff-4a8d-b2aa-580f3b22273f',
+                         volume_backup_size_samples[0].resource_id)

ceilometer/volume/cinder.py

@@ -0,0 +1,111 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+"""Common code for working with volumes
+"""
+
+from __future__ import absolute_import
+
+from ceilometer.agent import plugin_base
+from ceilometer import sample
+
+
+class _Base(plugin_base.PollsterBase):
+    def extract_metadata(self, obj):
+        return dict((k, getattr(obj, k)) for k in self.FIELDS)
+
+
+class VolumeSizePollster(_Base):
+    @property
+    def default_discovery(self):
+        return 'volumes'
+
+    FIELDS = ['name',
+              'status',
+              'volume_type',
+              'os-vol-host-attr:host',
+              'migration_status',
+              'attachments',
+              'snapshot_id',
+              'source_volid']
+
+    def get_samples(self, manager, cache, resources):
+        for volume in resources:
+            yield sample.Sample(
+                name='volume.size',
+                type=sample.TYPE_GAUGE,
+                unit='GB',
+                volume=volume.size,
+                user_id=volume.user_id,
+                project_id=getattr(volume,
+                                   'os-vol-tenant-attr:tenant_id'),
+                resource_id=volume.id,
+                resource_metadata=self.extract_metadata(volume),
+            )
+
+
+class VolumeSnapshotSize(_Base):
+    @property
+    def default_discovery(self):
+        return 'volume_snapshots'
+
+    FIELDS = ['name',
+              'volume_id',
+              'status',
+              'description',
+              'metadata',
+              'os-extended-snapshot-attributes:progress',
+              ]
+
+    def get_samples(self, manager, cache, resources):
+        for snapshot in resources:
+            yield sample.Sample(
+                name='volume.snapshot.size',
+                type=sample.TYPE_GAUGE,
+                unit='GB',
+                volume=snapshot.size,
+                user_id=None,
+                project_id=getattr(
+                    snapshot,
+                    'os-extended-snapshot-attributes:project_id'),
+                resource_id=snapshot.id,
+                resource_metadata=self.extract_metadata(snapshot),
+            )
+
+
+class VolumeBackupSize(_Base):
+    @property
+    def default_discovery(self):
+        return 'volume_backups'
+
+    FIELDS = ['name',
+              'object_count',
+              'container',
+              'volume_id',
+              'status',
+              'description']
+
+    def get_samples(self, manager, cache, resources):
+        for backup in resources:
+            yield sample.Sample(
+                name='volume.backup.size',
+                type=sample.TYPE_GAUGE,
+                unit='GB',
+                volume=backup.size,
+                user_id=None,
+                # TODO(liusheng): the tenant attribute isn't supported now,
+                # see: https://blueprints.launchpad.net/cinder/+spec/
+                # backup-tenant-attribute-support
+                project_id=None,
+                resource_id=backup.id,
+                resource_metadata=self.extract_metadata(backup),
+            )

ceilometer/volume/discovery.py

@@ -0,0 +1,62 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from cinderclient import client as cinder_client
+from oslo_config import cfg
+
+from ceilometer.agent import plugin_base
+from ceilometer import keystone_client
+
+SERVICE_OPTS = [
+    cfg.StrOpt('cinderv2',
+               default='volumev2',
+               help='Cinder V2 service type.'),
+]
+
+cfg.CONF.register_opts(SERVICE_OPTS, group='service_types')
+cfg.CONF.import_group('service_credentials', 'ceilometer.keystone_client')
+
+
+class _BaseDiscovery(plugin_base.DiscoveryBase):
+    def __init__(self, conf):
+        super(_BaseDiscovery, self).__init__(conf)
+        creds = conf.service_credentials
+        self.client = cinder_client.Client(
+            version='2',
+            session=keystone_client.get_session(conf),
+            region_name=creds.region_name,
+            interface=creds.interface,
+            service_type=conf.service_types.cinderv2
+        )
+
+
+class VolumeDiscovery(_BaseDiscovery):
+    def discover(self, manager, param=None):
+        """Discover volume resources to monitor."""
+
+        return self.client.volumes.list(search_opts={'all_tenants': True})
+
+
+class VolumeSnapshotsDiscovery(_BaseDiscovery):
+    def discover(self, manager, param=None):
+        """Discover snapshot resources to monitor."""
+
+        return self.client.volume_snapshots.list(
+            search_opts={'all_tenants': True})
+
+
+class VolumeBackupsDiscovery(_BaseDiscovery):
+    def discover(self, manager, param=None):
+        """Discover volume resources to monitor."""
+
+        return self.client.backups.list(search_opts={'all_tenants': True})

releasenotes/notes/support-cinder-volume-snapshot-backup-metering-d0a93b86bd53e803.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - Add support of metering the size of cinder volume/snapshot/backup. Like
+    other meters, these are useful for billing system.

requirements.txt

@@ -35,6 +35,7 @@ keystoneauth1>=2.1.0 # Apache-2.0
 python-neutronclient>=4.2.0 # Apache-2.0
 python-novaclient!=2.33.0,>=2.29.0 # Apache-2.0
 python-swiftclient>=2.2.0 # Apache-2.0
+python-cinderclient>=1.6.0,!=1.7.0,!=1.7.1  # Apache-2.0
 PyYAML>=3.1.0 # MIT
 requests!=2.9.0,>=2.8.1 # Apache-2.0
 six>=1.9.0 # MIT

setup.cfg

@@ -71,6 +71,9 @@ ceilometer.discover.central =
     tripleo_overcloud_nodes = ceilometer.hardware.discovery:NodesDiscoveryTripleO
     fip_services = ceilometer.network.services.discovery:FloatingIPDiscovery
     images = ceilometer.image.discovery:ImagesDiscovery
+    volumes = ceilometer.volume.discovery:VolumeDiscovery
+    volume_snapshots = ceilometer.volume.discovery:VolumeSnapshotsDiscovery
+    volume_backups = ceilometer.volume.discovery:VolumeBackupsDiscovery
 
 ceilometer.discover.ipmi =
     local_node = ceilometer.agent.discovery.localnode:LocalNodeDiscovery
@@ -193,6 +196,9 @@ ceilometer.poll.central =
     network.services.vpn.connections = ceilometer.network.services.vpnaas:IPSecConnectionsPollster
     network.services.firewall = ceilometer.network.services.fwaas:FirewallPollster
     network.services.firewall.policy = ceilometer.network.services.fwaas:FirewallPolicyPollster
+    volume.size = ceilometer.volume.cinder:VolumeSizePollster
+    volume.snapshot.size = ceilometer.volume.cinder:VolumeSnapshotSize
+    volume.backup.size = ceilometer.volume.cinder:VolumeBackupSize
 
 ceilometer.builder.poll.central =
     hardware.snmp = ceilometer.hardware.pollsters.generic:GenericHardwareDeclarativePollster