Merge "add new meters about some perf events"

ceilometer/compute/pollsters/perf.py

@@ -0,0 +1,128 @@
+# Copyright 2016 Intel
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import abc
+import collections
+
+from oslo_log import log
+
+import ceilometer
+from ceilometer.agent import plugin_base
+from ceilometer.compute import pollsters
+from ceilometer.compute.pollsters import util
+from ceilometer.compute.virt import inspector as virt_inspector
+from ceilometer.i18n import _LE, _LW
+from ceilometer import sample
+
+LOG = log.getLogger(__name__)
+
+
+PerfEventsData = collections.namedtuple('PerfEventsData',
+                                        ['cpu_cycles', 'instructions',
+                                         'cache_references', 'cache_misses'])
+
+
+class _PerfEventsPollster(pollsters.BaseComputePollster):
+
+    CACHE_KEY_MEMORY_BANDWIDTH = 'perf-events'
+
+    def _populate_cache(self, inspector, cache, instance):
+        i_cache = cache.setdefault(self.CACHE_KEY_MEMORY_BANDWIDTH, {})
+        if instance.id not in i_cache:
+            perf_events = self.inspector.inspect_perf_events(
+                instance, self._inspection_duration)
+            i_cache[instance.id] = PerfEventsData(
+                perf_events.cpu_cycles,
+                perf_events.instructions,
+                perf_events.cache_references,
+                perf_events.cache_misses,
+            )
+        return i_cache[instance.id]
+
+    @abc.abstractmethod
+    def _get_samples(self, instance, c_data):
+        """Return one or more Samples."""
+
+    def _get_sample_total_and_local(self, instance, _name, _unit,
+                                    c_data, _element):
+        """Total / local Pollster and return one Sample"""
+        return [util.make_sample_from_instance(
+            instance,
+            name=_name,
+            type=sample.TYPE_GAUGE,
+            unit=_unit,
+            volume=getattr(c_data, _element),
+        )]
+
+    def get_samples(self, manager, cache, resources):
+        self._inspection_duration = self._record_poll_time()
+        for instance in resources:
+            try:
+                c_data = self._populate_cache(
+                    self.inspector,
+                    cache,
+                    instance,
+                )
+                for s in self._get_samples(instance, c_data):
+                    yield s
+            except virt_inspector.InstanceNotFoundException as err:
+                # Instance was deleted while getting samples. Ignore it.
+                LOG.debug('Exception while getting samples %s', err)
+            except virt_inspector.InstanceShutOffException as e:
+                LOG.debug('Instance %(instance_id)s was shut off while '
+                          'getting samples of %(pollster)s: %(exc)s',
+                          {'instance_id': instance.id,
+                           'pollster': self.__class__.__name__, 'exc': e})
+            except virt_inspector.NoDataException as e:
+                LOG.warning(_LW('Cannot inspect data of %(pollster)s for '
+                                '%(instance_id)s, non-fatal reason: %(exc)s'),
+                            {'pollster': self.__class__.__name__,
+                             'instance_id': instance.id, 'exc': e})
+                raise plugin_base.PollsterPermanentError(resources)
+            except ceilometer.NotImplementedError:
+                # Selected inspector does not implement this pollster.
+                LOG.debug('Obtaining perf events is not implemented'
+                          ' for %s', self.inspector.__class__.__name__)
+            except Exception as err:
+                LOG.exception(_LE('Could not get perf events for '
+                                  '%(id)s: %(e)s'), {'id': instance.id,
+                                                     'e': err})
+
+
+class PerfEventsCPUCyclesPollster(_PerfEventsPollster):
+
+    def _get_samples(self, instance, c_data):
+        return self._get_sample_total_and_local(
+            instance, 'perf.cpu.cycles', '', c_data, 'cpu_cycles')
+
+
+class PerfEventsInstructionsPollster(_PerfEventsPollster):
+
+    def _get_samples(self, instance, c_data):
+        return self._get_sample_total_and_local(
+            instance, 'perf.instructions', '', c_data, 'instructions')
+
+
+class PerfEventsCacheReferencesPollster(_PerfEventsPollster):
+
+    def _get_samples(self, instance, c_data):
+        return self._get_sample_total_and_local(
+            instance, 'perf.cache.references', '', c_data, 'cache_references')
+
+
+class PerfEventsCacheMissesPollster(_PerfEventsPollster):
+
+    def _get_samples(self, instance, c_data):
+        return self._get_sample_total_and_local(
+            instance, 'perf.cache.misses', '', c_data, 'cache_misses')

ceilometer/compute/virt/inspector.py

@@ -88,6 +88,19 @@ MemoryResidentStats = collections.namedtuple('MemoryResidentStats',
 MemoryBandwidthStats = collections.namedtuple('MemoryBandwidthStats',
                                               ['total', 'local'])
 
+
+# Named tuple representing perf events statistics.
+#
+# cpu_cycles: the number of cpu cycles one instruction needs
+# instructions: the count of instructions
+# cache_references: the count of cache hits
+# cache_misses: the count of caches misses
+#
+PerfEventsStats = collections.namedtuple('PerfEventsStats',
+                                         ['cpu_cycles', 'instructions',
+                                          'cache_references', 'cache_misses'])
+
+
 # Named tuple representing vNICs.
 #
 # name: the name of the vNIC
@@ -339,6 +352,16 @@ class Inspector(object):
         """
         raise ceilometer.NotImplementedError
 
+    def inspect_perf_events(self, instance, duration=None):
+        """Inspect the perf events statistics for an instance.
+
+        :param instance: the target instance
+        :param duration: the last 'n' seconds, over which the value should be
+               inspected
+        :return:
+        """
+        raise ceilometer.NotImplementedError
+
 
 def get_hypervisor_inspector():
     try:

ceilometer/compute/virt/libvirt/inspector.py

@@ -22,7 +22,7 @@ import six
 
 from ceilometer.compute.pollsters import util
 from ceilometer.compute.virt import inspector as virt_inspector
-from ceilometer.i18n import _LW, _
+from ceilometer.i18n import _LW, _LE, _
 
 libvirt = None
 
@@ -279,3 +279,31 @@ class LibvirtInspector(virt_inspector.Inspector):
                     'can not get info from libvirt: %(error)s') % {
                 'instance_uuid': instance.id, 'error': e}
             raise virt_inspector.NoDataException(msg)
+
+    def inspect_perf_events(self, instance, duration=None):
+        domain = self._get_domain_not_shut_off_or_raise(instance)
+
+        try:
+            stats = self.connection.domainListGetStats(
+                [domain], libvirt.VIR_DOMAIN_STATS_PERF)
+            perf = stats[0][1]
+            return virt_inspector.PerfEventsStats(
+                cpu_cycles=perf["perf.cpu_cycles"],
+                instructions=perf["perf.instructions"],
+                cache_references=perf["perf.cache_references"],
+                cache_misses=perf["perf.cache_misses"])
+        except AttributeError as e:
+            msg = _LE('Perf is not supported by current version of libvirt, '
+                      'and failed to inspect perf events of '
+                      '%(instance_uuid)s, can not get info from libvirt: '
+                      '%(error)s') % {
+                'instance_uuid': instance.id, 'error': e}
+            raise virt_inspector.NoDataException(msg)
+        # domainListGetStats might launch an exception if the method or
+        # mbmt/mbml perf event is not supported by the underlying hypervisor
+        # being used by libvirt.
+        except libvirt.libvirtError as e:
+            msg = _LE('Failed to inspect perf events of %(instance_uuid)s, '
+                      'can not get info from libvirt: %(error)s') % {
+                'instance_uuid': instance.id, 'error': e}
+            raise virt_inspector.NoDataException(msg)

ceilometer/tests/unit/compute/pollsters/test_perf.py

@@ -0,0 +1,100 @@
+# Copyright 2016 Intel
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+
+from ceilometer.agent import manager
+from ceilometer.agent import plugin_base
+from ceilometer.compute.pollsters import perf
+from ceilometer.compute.virt import inspector as virt_inspector
+from ceilometer.tests.unit.compute.pollsters import base
+
+
+class TestPerfEventsPollster(base.TestPollsterBase):
+
+    def setUp(self):
+        super(TestPerfEventsPollster, self).setUp()
+
+    @mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
+    def test_get_samples(self):
+        fake_value = virt_inspector.PerfEventsStats(cpu_cycles=7259361,
+                                                    instructions=8815623,
+                                                    cache_references=74184,
+                                                    cache_misses=16737)
+
+        def inspect_perf_events(instance, duration):
+            return fake_value
+
+        self.inspector.inspect_perf_events = mock.Mock(
+            side_effect=inspect_perf_events)
+        mgr = manager.AgentManager()
+
+        def _check_perf_events_cpu_cycles(expected_usage):
+            pollster = perf.PerfEventsCPUCyclesPollster()
+
+            samples = list(pollster.get_samples(mgr, {}, [self.instance]))
+            self.assertEqual(1, len(samples))
+            self.assertEqual(set(['perf.cpu.cycles']),
+                             set([s.name for s in samples]))
+            self.assertEqual(expected_usage, samples[0].volume)
+
+        def _check_perf_events_instructions(expected_usage):
+            pollster = perf.PerfEventsInstructionsPollster()
+
+            samples = list(pollster.get_samples(mgr, {}, [self.instance]))
+            self.assertEqual(1, len(samples))
+            self.assertEqual(set(['perf.instructions']),
+                             set([s.name for s in samples]))
+            self.assertEqual(expected_usage, samples[0].volume)
+
+        def _check_perf_events_cache_references(expected_usage):
+            pollster = perf.PerfEventsCacheReferencesPollster()
+
+            samples = list(pollster.get_samples(mgr, {}, [self.instance]))
+            self.assertEqual(1, len(samples))
+            self.assertEqual(set(['perf.cache.references']),
+                             set([s.name for s in samples]))
+            self.assertEqual(expected_usage, samples[0].volume)
+
+        def _check_perf_events_cache_misses(expected_usage):
+            pollster = perf.PerfEventsCacheMissesPollster()
+
+            samples = list(pollster.get_samples(mgr, {}, [self.instance]))
+            self.assertEqual(1, len(samples))
+            self.assertEqual(set(['perf.cache.misses']),
+                             set([s.name for s in samples]))
+            self.assertEqual(expected_usage, samples[0].volume)
+
+        _check_perf_events_cpu_cycles(7259361)
+        _check_perf_events_instructions(8815623)
+        _check_perf_events_cache_references(74184)
+        _check_perf_events_cache_misses(16737)
+
+    @mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
+    def test_get_samples_with_empty_stats(self):
+
+        def inspect_perf_events(instance, duration):
+            raise virt_inspector.NoDataException()
+
+        self.inspector.inspect_perf_events = mock.Mock(
+            side_effect=inspect_perf_events)
+
+        mgr = manager.AgentManager()
+        pollster = perf.PerfEventsCPUCyclesPollster()
+
+        def all_samples():
+            return list(pollster.get_samples(mgr, {}, [self.instance]))
+
+        self.assertRaises(plugin_base.PollsterPermanentError,
+                          all_samples)

ceilometer/tests/unit/compute/virt/libvirt/test_inspector.py

@@ -386,6 +386,25 @@ class TestLibvirtInspection(base.BaseTestCase):
                     self.assertEqual(1892352, mb.total)
                     self.assertEqual(1802240, mb.local)
 
+    def test_inspect_perf_events(self):
+        fake_stats = [({}, {'perf.cpu_cycles': 7259361,
+                            'perf.instructions': 8815623,
+                            'perf.cache_references': 74184,
+                            'perf.cache_misses': 16737})]
+        connection = self.inspector.connection
+        with mock.patch.object(connection, 'lookupByUUIDString',
+                               return_value=self.domain):
+            with mock.patch.object(self.domain, 'info',
+                                   return_value=(0, 0, 51200,
+                                                 2, 999999)):
+                with mock.patch.object(connection, 'domainListGetStats',
+                                       return_value=fake_stats):
+                    pe = self.inspector.inspect_perf_events(self.instance)
+                    self.assertEqual(7259361, pe.cpu_cycles)
+                    self.assertEqual(8815623, pe.instructions)
+                    self.assertEqual(74184, pe.cache_references)
+                    self.assertEqual(16737, pe.cache_misses)
+
 
 class TestLibvirtInspectionWithError(base.BaseTestCase):
 

releasenotes/notes/perf-events-meter-b06c2a915c33bfaf.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - Add four new meters, including perf.cpu.cycles for the number
+    of cpu cycles one instruction needs, perf.instructions for the
+    count of instructions, perf.cache_references for the count of
+    cache hits and cache_misses for the count of caches misses.

setup.cfg

@@ -136,6 +136,10 @@ ceilometer.poll.compute =
     disk.device.capacity = ceilometer.compute.pollsters.disk:PerDeviceCapacityPollster
     disk.device.allocation = ceilometer.compute.pollsters.disk:PerDeviceAllocationPollster
     disk.device.usage = ceilometer.compute.pollsters.disk:PerDevicePhysicalPollster
+    perf.cpu.cycles = ceilometer.compute.pollsters.perf:PerfEventsCPUCyclesPollster
+    perf.instructions = ceilometer.compute.pollsters.perf:PerfEventsInstructionsPollster
+    perf.cache.references = ceilometer.compute.pollsters.perf:PerfEventsCacheReferencesPollster
+    perf.cache.misses = ceilometer.compute.pollsters.perf:PerfEventsCacheMissesPollster
 
 ceilometer.poll.ipmi =
     hardware.ipmi.node.power = ceilometer.ipmi.pollsters.node:PowerPollster