add new meters about some perf events

We need more statistics, including IPC, cache references and
cache misses, to do performance tuning. This patch introduces
four new meters to get them.

Change-Id: Icb21eeed0e004450556176d01520784acd98002e

ceilometer/compute/pollsters/perf.py

@@ -0,0 +1,128 @@
+# Copyright 2016 Intel
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import abc
+import collections
+
+from oslo_log import log
+
+import ceilometer
+from ceilometer.agent import plugin_base
+from ceilometer.compute import pollsters
+from ceilometer.compute.pollsters import util
+from ceilometer.compute.virt import inspector as virt_inspector
+from ceilometer.i18n import _LE, _LW
+from ceilometer import sample
+
+LOG = log.getLogger(__name__)
+
+
+PerfEventsData = collections.namedtuple('PerfEventsData',
+                                        ['cpu_cycles', 'instructions',
+                                         'cache_references', 'cache_misses'])
+
+
+class _PerfEventsPollster(pollsters.BaseComputePollster):
+
+    CACHE_KEY_MEMORY_BANDWIDTH = 'perf-events'
+
+    def _populate_cache(self, inspector, cache, instance):
+        i_cache = cache.setdefault(self.CACHE_KEY_MEMORY_BANDWIDTH, {})
+        if instance.id not in i_cache:
+            perf_events = self.inspector.inspect_perf_events(
+                instance, self._inspection_duration)
+            i_cache[instance.id] = PerfEventsData(
+                perf_events.cpu_cycles,
+                perf_events.instructions,
+                perf_events.cache_references,
+                perf_events.cache_misses,
+            )
+        return i_cache[instance.id]
+
+    @abc.abstractmethod
+    def _get_samples(self, instance, c_data):
+        """Return one or more Samples."""
+
+    def _get_sample_total_and_local(self, instance, _name, _unit,
+                                    c_data, _element):
+        """Total / local Pollster and return one Sample"""
+        return [util.make_sample_from_instance(
+            instance,
+            name=_name,
+            type=sample.TYPE_GAUGE,
+            unit=_unit,
+            volume=getattr(c_data, _element),
+        )]
+
+    def get_samples(self, manager, cache, resources):
+        self._inspection_duration = self._record_poll_time()
+        for instance in resources:
+            try:
+                c_data = self._populate_cache(
+                    self.inspector,
+                    cache,
+                    instance,
+                )
+                for s in self._get_samples(instance, c_data):
+                    yield s
+            except virt_inspector.InstanceNotFoundException as err:
+                # Instance was deleted while getting samples. Ignore it.
+                LOG.debug('Exception while getting samples %s', err)
+            except virt_inspector.InstanceShutOffException as e:
+                LOG.debug('Instance %(instance_id)s was shut off while '
+                          'getting samples of %(pollster)s: %(exc)s',
+                          {'instance_id': instance.id,
+                           'pollster': self.__class__.__name__, 'exc': e})
+            except virt_inspector.NoDataException as e:
+                LOG.warning(_LW('Cannot inspect data of %(pollster)s for '
+                                '%(instance_id)s, non-fatal reason: %(exc)s'),
+                            {'pollster': self.__class__.__name__,
+                             'instance_id': instance.id, 'exc': e})
+                raise plugin_base.PollsterPermanentError(resources)
+            except ceilometer.NotImplementedError:
+                # Selected inspector does not implement this pollster.
+                LOG.debug('Obtaining perf events is not implemented'
+                          ' for %s', self.inspector.__class__.__name__)
+            except Exception as err:
+                LOG.exception(_LE('Could not get perf events for '
+                                  '%(id)s: %(e)s'), {'id': instance.id,
+                                                     'e': err})
+
+
+class PerfEventsCPUCyclesPollster(_PerfEventsPollster):
+
+    def _get_samples(self, instance, c_data):
+        return self._get_sample_total_and_local(
+            instance, 'perf.cpu.cycles', '', c_data, 'cpu_cycles')
+
+
+class PerfEventsInstructionsPollster(_PerfEventsPollster):
+
+    def _get_samples(self, instance, c_data):
+        return self._get_sample_total_and_local(
+            instance, 'perf.instructions', '', c_data, 'instructions')
+
+
+class PerfEventsCacheReferencesPollster(_PerfEventsPollster):
+
+    def _get_samples(self, instance, c_data):
+        return self._get_sample_total_and_local(
+            instance, 'perf.cache.references', '', c_data, 'cache_references')
+
+
+class PerfEventsCacheMissesPollster(_PerfEventsPollster):
+
+    def _get_samples(self, instance, c_data):
+        return self._get_sample_total_and_local(
+            instance, 'perf.cache.misses', '', c_data, 'cache_misses')

ceilometer/compute/virt/inspector.py

@@ -88,6 +88,19 @@ MemoryResidentStats = collections.namedtuple('MemoryResidentStats',
 MemoryBandwidthStats = collections.namedtuple('MemoryBandwidthStats',
                                               ['total', 'local'])
 
+
+# Named tuple representing perf events statistics.
+#
+# cpu_cycles: the number of cpu cycles one instruction needs
+# instructions: the count of instructions
+# cache_references: the count of cache hits
+# cache_misses: the count of caches misses
+#
+PerfEventsStats = collections.namedtuple('PerfEventsStats',
+                                         ['cpu_cycles', 'instructions',
+                                          'cache_references', 'cache_misses'])
+
+
 # Named tuple representing vNICs.
 #
 # name: the name of the vNIC
@@ -339,6 +352,16 @@ class Inspector(object):
         """
         raise ceilometer.NotImplementedError
 
+    def inspect_perf_events(self, instance, duration=None):
+        """Inspect the perf events statistics for an instance.
+
+        :param instance: the target instance
+        :param duration: the last 'n' seconds, over which the value should be
+               inspected
+        :return:
+        """
+        raise ceilometer.NotImplementedError
+
 
 def get_hypervisor_inspector():
     try:

ceilometer/compute/virt/libvirt/inspector.py

@@ -22,7 +22,7 @@ import six
 
 from ceilometer.compute.pollsters import util
 from ceilometer.compute.virt import inspector as virt_inspector
-from ceilometer.i18n import _LW, _
+from ceilometer.i18n import _LW, _LE, _
 
 libvirt = None
 
@@ -279,3 +279,31 @@ class LibvirtInspector(virt_inspector.Inspector):
                     'can not get info from libvirt: %(error)s') % {
                 'instance_uuid': instance.id, 'error': e}
             raise virt_inspector.NoDataException(msg)
+
+    def inspect_perf_events(self, instance, duration=None):
+        domain = self._get_domain_not_shut_off_or_raise(instance)
+
+        try:
+            stats = self.connection.domainListGetStats(
+                [domain], libvirt.VIR_DOMAIN_STATS_PERF)
+            perf = stats[0][1]
+            return virt_inspector.PerfEventsStats(
+                cpu_cycles=perf["perf.cpu_cycles"],
+                instructions=perf["perf.instructions"],
+                cache_references=perf["perf.cache_references"],
+                cache_misses=perf["perf.cache_misses"])
+        except AttributeError as e:
+            msg = _LE('Perf is not supported by current version of libvirt, '
+                      'and failed to inspect perf events of '
+                      '%(instance_uuid)s, can not get info from libvirt: '
+                      '%(error)s') % {
+                'instance_uuid': instance.id, 'error': e}
+            raise virt_inspector.NoDataException(msg)
+        # domainListGetStats might launch an exception if the method or
+        # mbmt/mbml perf event is not supported by the underlying hypervisor
+        # being used by libvirt.
+        except libvirt.libvirtError as e:
+            msg = _LE('Failed to inspect perf events of %(instance_uuid)s, '
+                      'can not get info from libvirt: %(error)s') % {
+                'instance_uuid': instance.id, 'error': e}
+            raise virt_inspector.NoDataException(msg)

ceilometer/tests/unit/compute/pollsters/test_perf.py

@@ -0,0 +1,100 @@
+# Copyright 2016 Intel
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+
+from ceilometer.agent import manager
+from ceilometer.agent import plugin_base
+from ceilometer.compute.pollsters import perf
+from ceilometer.compute.virt import inspector as virt_inspector
+from ceilometer.tests.unit.compute.pollsters import base
+
+
+class TestPerfEventsPollster(base.TestPollsterBase):
+
+    def setUp(self):
+        super(TestPerfEventsPollster, self).setUp()
+
+    @mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
+    def test_get_samples(self):
+        fake_value = virt_inspector.PerfEventsStats(cpu_cycles=7259361,
+                                                    instructions=8815623,
+                                                    cache_references=74184,
+                                                    cache_misses=16737)
+
+        def inspect_perf_events(instance, duration):
+            return fake_value
+
+        self.inspector.inspect_perf_events = mock.Mock(
+            side_effect=inspect_perf_events)
+        mgr = manager.AgentManager()
+
+        def _check_perf_events_cpu_cycles(expected_usage):
+            pollster = perf.PerfEventsCPUCyclesPollster()
+
+            samples = list(pollster.get_samples(mgr, {}, [self.instance]))
+            self.assertEqual(1, len(samples))
+            self.assertEqual(set(['perf.cpu.cycles']),
+                             set([s.name for s in samples]))
+            self.assertEqual(expected_usage, samples[0].volume)
+
+        def _check_perf_events_instructions(expected_usage):
+            pollster = perf.PerfEventsInstructionsPollster()
+
+            samples = list(pollster.get_samples(mgr, {}, [self.instance]))
+            self.assertEqual(1, len(samples))
+            self.assertEqual(set(['perf.instructions']),
+                             set([s.name for s in samples]))
+            self.assertEqual(expected_usage, samples[0].volume)
+
+        def _check_perf_events_cache_references(expected_usage):
+            pollster = perf.PerfEventsCacheReferencesPollster()
+
+            samples = list(pollster.get_samples(mgr, {}, [self.instance]))
+            self.assertEqual(1, len(samples))
+            self.assertEqual(set(['perf.cache.references']),
+                             set([s.name for s in samples]))
+            self.assertEqual(expected_usage, samples[0].volume)
+
+        def _check_perf_events_cache_misses(expected_usage):
+            pollster = perf.PerfEventsCacheMissesPollster()
+
+            samples = list(pollster.get_samples(mgr, {}, [self.instance]))
+            self.assertEqual(1, len(samples))
+            self.assertEqual(set(['perf.cache.misses']),
+                             set([s.name for s in samples]))
+            self.assertEqual(expected_usage, samples[0].volume)
+
+        _check_perf_events_cpu_cycles(7259361)
+        _check_perf_events_instructions(8815623)
+        _check_perf_events_cache_references(74184)
+        _check_perf_events_cache_misses(16737)
+
+    @mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
+    def test_get_samples_with_empty_stats(self):
+
+        def inspect_perf_events(instance, duration):
+            raise virt_inspector.NoDataException()
+
+        self.inspector.inspect_perf_events = mock.Mock(
+            side_effect=inspect_perf_events)
+
+        mgr = manager.AgentManager()
+        pollster = perf.PerfEventsCPUCyclesPollster()
+
+        def all_samples():
+            return list(pollster.get_samples(mgr, {}, [self.instance]))
+
+        self.assertRaises(plugin_base.PollsterPermanentError,
+                          all_samples)

ceilometer/tests/unit/compute/virt/libvirt/test_inspector.py

@@ -386,6 +386,25 @@ class TestLibvirtInspection(base.BaseTestCase):
                     self.assertEqual(1892352, mb.total)
                     self.assertEqual(1802240, mb.local)
 
+    def test_inspect_perf_events(self):
+        fake_stats = [({}, {'perf.cpu_cycles': 7259361,
+                            'perf.instructions': 8815623,
+                            'perf.cache_references': 74184,
+                            'perf.cache_misses': 16737})]
+        connection = self.inspector.connection
+        with mock.patch.object(connection, 'lookupByUUIDString',
+                               return_value=self.domain):
+            with mock.patch.object(self.domain, 'info',
+                                   return_value=(0, 0, 51200,
+                                                 2, 999999)):
+                with mock.patch.object(connection, 'domainListGetStats',
+                                       return_value=fake_stats):
+                    pe = self.inspector.inspect_perf_events(self.instance)
+                    self.assertEqual(7259361, pe.cpu_cycles)
+                    self.assertEqual(8815623, pe.instructions)
+                    self.assertEqual(74184, pe.cache_references)
+                    self.assertEqual(16737, pe.cache_misses)
+
 
 class TestLibvirtInspectionWithError(base.BaseTestCase):
 

releasenotes/notes/perf-events-meter-b06c2a915c33bfaf.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - Add four new meters, including perf.cpu.cycles for the number
+    of cpu cycles one instruction needs, perf.instructions for the
+    count of instructions, perf.cache_references for the count of
+    cache hits and cache_misses for the count of caches misses.

setup.cfg

@@ -124,6 +124,10 @@ ceilometer.poll.compute =
     disk.device.capacity = ceilometer.compute.pollsters.disk:PerDeviceCapacityPollster
     disk.device.allocation = ceilometer.compute.pollsters.disk:PerDeviceAllocationPollster
     disk.device.usage = ceilometer.compute.pollsters.disk:PerDevicePhysicalPollster
+    perf.cpu.cycles = ceilometer.compute.pollsters.perf:PerfEventsCPUCyclesPollster
+    perf.instructions = ceilometer.compute.pollsters.perf:PerfEventsInstructionsPollster
+    perf.cache.references = ceilometer.compute.pollsters.perf:PerfEventsCacheReferencesPollster
+    perf.cache.misses = ceilometer.compute.pollsters.perf:PerfEventsCacheMissesPollster
 
 ceilometer.poll.ipmi =
     hardware.ipmi.node.power = ceilometer.ipmi.pollsters.node:PowerPollster