Merge "Changes for networking metrics support for vSphere"

ceilometer/compute/pollsters/net.py

@@ -64,37 +64,73 @@ class _Base(plugin.ComputePollster):
 
     CACHE_KEY_VNIC = 'vnics'
 
-    def _get_vnics_for_instance(self, cache, inspector, instance_name):
+    def _get_vnic_info(self, inspector, instance):
+        instance_name = util.instance_name(instance)
+        return inspector.inspect_vnics(instance_name)
+
+    def _get_rx_info(self, info):
+        return info.rx_bytes
+
+    def _get_tx_info(self, info):
+        return info.tx_bytes
+
+    def _get_vnics_for_instance(self, cache, inspector, instance):
+        instance_name = util.instance_name(instance)
         i_cache = cache.setdefault(self.CACHE_KEY_VNIC, {})
         if instance_name not in i_cache:
             i_cache[instance_name] = list(
-                inspector.inspect_vnics(instance_name)
+                self._get_vnic_info(inspector, instance)
             )
         return i_cache[instance_name]
 
     def get_samples(self, manager, cache, resources):
         for instance in resources:
             instance_name = util.instance_name(instance)
-            LOG.info(_('checking instance %s'), instance.id)
+            LOG.debug(_('checking net info for instance %s'), instance.id)
             try:
                 vnics = self._get_vnics_for_instance(
                     cache,
                     manager.inspector,
-                    instance_name,
+                    instance,
                 )
                 for vnic, info in vnics:
-                    LOG.info(self.NET_USAGE_MESSAGE, instance_name,
+                    LOG.debug(self.NET_USAGE_MESSAGE, instance_name,
-                             vnic.name, info.rx_bytes, info.tx_bytes)
+                              vnic.name, self._get_rx_info(info),
+                              self._get_tx_info(info))
                     yield self._get_sample(instance, vnic, info)
             except virt_inspector.InstanceNotFoundException as err:
                 # Instance was deleted while getting samples. Ignore it.
                 LOG.debug(_('Exception while getting samples %s'), err)
+            except NotImplementedError:
+                # Selected inspector does not implement this pollster.
+                LOG.debug(_('%(inspector)s does not provide data for '
+                            ' %(pollster)s'), ({
+                          'inspector': manager.inspector.__class__.__name__,
+                          'pollster': self.__class__.__name__}))
             except Exception as err:
                 LOG.warning(_('Ignoring instance %(name)s: %(error)s') % (
                             {'name': instance_name, 'error': err}))
                 LOG.exception(err)
 
 
+class _RateBase(_Base):
+
+    NET_USAGE_MESSAGE = ' '.join(["NETWORK RATE:", "%s %s:",
+                                 "read-bytes-rate=%d",
+                                 "write-bytes-rate=%d"])
+
+    CACHE_KEY_VNIC = 'vnic-rates'
+
+    def _get_vnic_info(self, inspector, instance):
+        return inspector.inspect_vnic_rates(instance)
+
+    def _get_rx_info(self, info):
+        return info.rx_bytes_rate
+
+    def _get_tx_info(self, info):
+        return info.tx_bytes_rate
+
+
 class IncomingBytesPollster(_Base):
 
     def _get_sample(self, instance, vnic, info):
@@ -145,3 +181,29 @@ class OutgoingPacketsPollster(_Base):
             volume=info.tx_packets,
             vnic_data=vnic,
         )
+
+
+class IncomingBytesRatePollster(_RateBase):
+
+    def _get_sample(self, instance, vnic, info):
+        return self.make_vnic_sample(
+            instance,
+            name='network.incoming.bytes.rate',
+            type=sample.TYPE_GAUGE,
+            unit='B/s',
+            volume=info.rx_bytes_rate,
+            vnic_data=vnic,
+        )
+
+
+class OutgoingBytesRatePollster(_RateBase):
+
+    def _get_sample(self, instance, vnic, info):
+        return self.make_vnic_sample(
+            instance,
+            name='network.outgoing.bytes.rate',
+            type=sample.TYPE_GAUGE,
+            unit='B/s',
+            volume=info.tx_bytes_rate,
+            vnic_data=vnic,
+        )

ceilometer/compute/virt/inspector.py

@@ -89,6 +89,15 @@ InterfaceStats = collections.namedtuple('InterfaceStats',
                                          'tx_bytes', 'tx_packets'])
 
 
+# Named tuple representing vNIC rate statistics.
+#
+# rx_bytes_rate: rate of received bytes
+# tx_bytes_rate: rate of transmitted bytes
+#
+InterfaceRateStats = collections.namedtuple('InterfaceRateStats',
+                                            ['rx_bytes_rate', 'tx_bytes_rate'])
+
+
 # Named tuple representing disks.
 #
 # device: the device name for the disk
@@ -154,6 +163,15 @@ class Inspector(object):
         """
         raise NotImplementedError()
 
+    def inspect_vnic_rates(self, instance):
+        """Inspect the vNIC rate statistics for an instance.
+
+        :param instance: the target instance
+        :return: for each vNIC, the rate of bytes & packets
+                 received and transmitted
+        """
+        raise NotImplementedError()
+
     def inspect_disks(self, instance_name):
         """Inspect the disk statistics for an instance.
 

ceilometer/compute/virt/vmware/inspector.py

@@ -51,6 +51,8 @@ cfg.CONF.register_opts(OPTS, group=opt_group)
 
 VC_AVERAGE_MEMORY_CONSUMED_CNTR = 'mem:consumed:average'
 VC_AVERAGE_CPU_CONSUMED_CNTR = 'cpu:usage:average'
+VC_NETWORK_RX_BYTES_COUNTER = 'net:bytesRx:average'
+VC_NETWORK_TX_BYTES_COUNTER = 'net:bytesTx:average'
 
 
 def get_api_session():
@@ -93,6 +95,38 @@ class VsphereInspector(virt_inspector.Inspector):
     def inspect_vnics(self, instance_name):
         raise NotImplementedError()
 
+    def inspect_vnic_rates(self, instance):
+        vm_moid = self._ops.get_vm_moid(instance.id)
+        if not vm_moid:
+            raise virt_inspector.InstanceNotFoundException(
+                _('VM %s not found in VMware Vsphere') % instance.id)
+
+        vnic_stats = {}
+        vnic_ids = set()
+
+        for net_counter in (VC_NETWORK_RX_BYTES_COUNTER,
+                            VC_NETWORK_TX_BYTES_COUNTER):
+            net_counter_id = self._ops.get_perf_counter_id(net_counter)
+            vnic_id_to_stats_map = \
+                self._ops.query_vm_device_stats(vm_moid, net_counter_id)
+            vnic_stats[net_counter] = vnic_id_to_stats_map
+            vnic_ids.update(vnic_id_to_stats_map.iterkeys())
+
+        for vnic_id in vnic_ids:
+            rx_bytes_rate = (vnic_stats[VC_NETWORK_RX_BYTES_COUNTER]
+                             .get(vnic_id, 0) / units.k)
+            tx_bytes_rate = (vnic_stats[VC_NETWORK_TX_BYTES_COUNTER]
+                             .get(vnic_id, 0) / units.k)
+
+            stats = virt_inspector.InterfaceRateStats(rx_bytes_rate,
+                                                      tx_bytes_rate)
+            interface = virt_inspector.Interface(
+                name=vnic_id,
+                mac=None,
+                fref=None,
+                parameters=None)
+            yield (interface, stats)
+
     def inspect_disks(self, instance_name):
         raise NotImplementedError()
 

ceilometer/tests/compute/pollsters/test_net.py

@@ -171,3 +171,89 @@ class TestNetPollsterCache(base.TestPollsterBase):
 
     def test_outgoing_packets(self):
         self._check_get_samples_cache(net.OutgoingPacketsPollster)
+
+
+class TestNetRatesPollster(base.TestPollsterBase):
+
+    def setUp(self):
+        super(TestNetRatesPollster, self).setUp()
+        self.vnic0 = virt_inspector.Interface(
+            name='vnet0',
+            fref='fa163e71ec6e',
+            mac='fa:16:3e:71:ec:6d',
+            parameters=dict(ip='10.0.0.2',
+                            projmask='255.255.255.0',
+                            projnet='proj1',
+                            dhcp_server='10.0.0.1'))
+        stats0 = virt_inspector.InterfaceRateStats(rx_bytes_rate=1L,
+                                                   tx_bytes_rate=2L)
+        self.vnic1 = virt_inspector.Interface(
+            name='vnet1',
+            fref='fa163e71ec6f',
+            mac='fa:16:3e:71:ec:6e',
+            parameters=dict(ip='192.168.0.3',
+                            projmask='255.255.255.0',
+                            projnet='proj2',
+                            dhcp_server='10.0.0.2'))
+        stats1 = virt_inspector.InterfaceRateStats(rx_bytes_rate=3L,
+                                                   tx_bytes_rate=4L)
+        self.vnic2 = virt_inspector.Interface(
+            name='vnet2',
+            fref=None,
+            mac='fa:18:4e:72:fc:7e',
+            parameters=dict(ip='192.168.0.4',
+                            projmask='255.255.255.0',
+                            projnet='proj3',
+                            dhcp_server='10.0.0.3'))
+        stats2 = virt_inspector.InterfaceRateStats(rx_bytes_rate=5L,
+                                                   tx_bytes_rate=6L)
+
+        vnics = [
+            (self.vnic0, stats0),
+            (self.vnic1, stats1),
+            (self.vnic2, stats2),
+        ]
+        self.inspector.inspect_vnic_rates = mock.Mock(return_value=vnics)
+
+    @mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
+    def _check_get_samples(self, factory, expected):
+        mgr = manager.AgentManager()
+        pollster = factory()
+        samples = list(pollster.get_samples(mgr, {}, [self.instance]))
+        self.assertEqual(3, len(samples))  # one for each nic
+        self.assertEqual(set([samples[0].name]),
+                         set([s.name for s in samples]))
+
+        def _verify_vnic_metering(ip, expected_volume, expected_rid):
+            match = [s for s in samples
+                     if s.resource_metadata['parameters']['ip'] == ip
+                     ]
+            self.assertEqual(1, len(match), 'missing ip %s' % ip)
+            self.assertEqual(expected_volume, match[0].volume)
+            self.assertEqual('gauge', match[0].type)
+            self.assertEqual(expected_rid, match[0].resource_id)
+
+        for ip, volume, rid in expected:
+            _verify_vnic_metering(ip, volume, rid)
+
+    def test_incoming_bytes_rate(self):
+        instance_name_id = "%s-%s" % (self.instance.name, self.instance.id)
+        self._check_get_samples(
+            net.IncomingBytesRatePollster,
+            [('10.0.0.2', 1L, self.vnic0.fref),
+             ('192.168.0.3', 3L, self.vnic1.fref),
+             ('192.168.0.4', 5L,
+              "%s-%s" % (instance_name_id, self.vnic2.name)),
+             ],
+        )
+
+    def test_outgoing_bytes(self):
+        instance_name_id = "%s-%s" % (self.instance.name, self.instance.id)
+        self._check_get_samples(
+            net.OutgoingBytesRatePollster,
+            [('10.0.0.2', 2L, self.vnic0.fref),
+             ('192.168.0.3', 4L, self.vnic1.fref),
+             ('192.168.0.4', 6L,
+              "%s-%s" % (instance_name_id, self.vnic2.name)),
+             ],
+        )

ceilometer/tests/compute/virt/vmware/test_inspector.py

@@ -80,3 +80,44 @@ class TestVsphereInspection(test.BaseTestCase):
             fake_cpu_util_value
         cpu_util_stat = self._inspector.inspect_cpu_util(fake_instance)
         self.assertEqual(fake_stat, cpu_util_stat)
+
+    def test_inspect_vnic_rates(self):
+
+        # construct test data
+        test_vm_moid = "vm-21"
+        vnic1 = "vnic-1"
+        vnic2 = "vnic-2"
+        counter_name_to_id_map = {
+            vsphere_inspector.VC_NETWORK_RX_BYTES_COUNTER: 1,
+            vsphere_inspector.VC_NETWORK_TX_BYTES_COUNTER: 2
+        }
+        counter_id_to_stats_map = {
+            1: {vnic1: 1000.0, vnic2: 3000.0},
+            2: {vnic1: 2000.0, vnic2: 4000.0},
+        }
+
+        def get_counter_id_side_effect(counter_full_name):
+            return counter_name_to_id_map[counter_full_name]
+
+        def query_stat_side_effect(vm_moid, counter_id):
+            # assert inputs
+            self.assertEqual(test_vm_moid, vm_moid)
+            self.assertTrue(counter_id in counter_id_to_stats_map)
+            return counter_id_to_stats_map[counter_id]
+
+        # configure vsphere operations mock with the test data
+        ops_mock = self._inspector._ops
+        ops_mock.get_vm_moid.return_value = test_vm_moid
+        ops_mock.get_perf_counter_id.side_effect = get_counter_id_side_effect
+        ops_mock.query_vm_device_stats.side_effect = \
+            query_stat_side_effect
+        result = self._inspector.inspect_vnic_rates(mock.MagicMock())
+
+        # validate result
+        expected_stats = {
+            vnic1: virt_inspector.InterfaceRateStats(1.0, 2.0),
+            vnic2: virt_inspector.InterfaceRateStats(3.0, 4.0)
+        }
+
+        for vnic, rates_info in result:
+            self.assertEqual(expected_stats[vnic.name], rates_info)

setup.cfg

@@ -78,6 +78,8 @@ ceilometer.poll.compute =
     network.incoming.packets = ceilometer.compute.pollsters.net:IncomingPacketsPollster
     network.outgoing.bytes = ceilometer.compute.pollsters.net:OutgoingBytesPollster
     network.outgoing.packets = ceilometer.compute.pollsters.net:OutgoingPacketsPollster
+    network.incoming.bytes.rate = ceilometer.compute.pollsters.net:IncomingBytesRatePollster
+    network.outgoing.bytes.rate = ceilometer.compute.pollsters.net:OutgoingBytesRatePollster
     instance = ceilometer.compute.pollsters.instance:InstancePollster
     instance_flavor = ceilometer.compute.pollsters.instance:InstanceFlavorPollster
     memory.usage = ceilometer.compute.pollsters.memory:MemoryUsagePollster