libvirt: add perf event support when create instance

Libvirt 1.3.3 has intergrated `perf`, which can be used as performance
statistics.

This patch enable perf event when create instances, the perf event
data can be collected by Ceilometer or other external monitor system.

Implement blueprint: support-perf-event

Co-Authored-By: Qiaowei Ren <qiaowei.ren@intel.com>

Change-Id: I2ffdabe40a7706cd5061a45f3e46c8245adb3b07

nova/conf/libvirt.py

@@ -216,7 +216,32 @@ libvirt_general_opts = [
                default=1,
                help='In a realtime host context vCPUs for guest will run in '
                     'that scheduling priority. Priority depends on the host '
-                    'kernel (usually 1-99)')
+                    'kernel (usually 1-99)'),
+    cfg.ListOpt('enabled_perf_events',
+               default=[],
+               help= """
+This is a performance event list which could be used as monitor. These events
+will be passed to libvirt domain xml while creating a new instances.
+Then event statistics data can be collected from libvirt.  The minimum
+libvirt version is 1.3.3.
+
+* Possible values:
+    A string list.
+    For example:
+    ``enabled_perf_events = cmt, mbml, mbmt``
+
+    The supported events list can be found in
+    https://libvirt.org/html/libvirt-libvirt-domain.html , which
+    you may need to search key words ``VIR_PERF_PARAM_*``
+
+* Services that use this:
+
+    ``nova-compute``
+
+* Related options:
+    None
+
+"""),
 ]
 
 libvirt_imagebackend_opts = [

nova/tests/unit/virt/libvirt/test_config.py

@@ -1990,6 +1990,32 @@ class LibvirtConfigGuestTest(LibvirtConfigBaseTest):
               </os>
             </domain>""")
 
+    def test_config_perf(self):
+        obj = config.LibvirtConfigGuest()
+        obj.virt_type = "kvm"
+        obj.memory = 100 * units.Mi
+        obj.vcpus = 2
+        obj.name = "perf"
+        obj.uuid = "f01cf68d-515c-4daf-b85f-ef1424d93bfc"
+        obj.os_type = "fake"
+        obj.perf_events = ['cmt', 'mbml']
+        xml = obj.to_xml()
+
+        self.assertXmlEqual(xml, """
+            <domain type="kvm">
+              <uuid>f01cf68d-515c-4daf-b85f-ef1424d93bfc</uuid>
+              <name>perf</name>
+              <memory>104857600</memory>
+              <vcpu>2</vcpu>
+              <os>
+                <type>fake</type>
+              </os>
+              <perf>
+                <event enabled="yes" name="cmt"/>
+                <event enabled="yes" name="mbml"/>
+              </perf>
+            </domain>""")
+
     def test_config_machine_type(self):
         obj = config.LibvirtConfigGuest()
         obj.virt_type = "kvm"
@@ -2054,6 +2080,19 @@ class LibvirtConfigGuestTest(LibvirtConfigBaseTest):
         self.assertEqual(obj.cpu.match, 'exact')
         self.assertEqual(obj.cpu.model, 'kvm64')
 
+    def test_ConfigGuest_parse_perf(self):
+        xmldoc = """ <domain>
+             <perf>
+               <event enabled="yes" name="cmt"/>
+               <event enabled="no" name="mbml"/>
+             </perf>
+                    </domain>
+               """
+        obj = config.LibvirtConfigGuest()
+        obj.parse_str(xmldoc)
+
+        self.assertEqual(['cmt'], obj.perf_events)
+
 
 class LibvirtConfigGuestSnapshotTest(LibvirtConfigBaseTest):
 

nova/tests/unit/virt/libvirt/test_driver.py

@@ -5276,6 +5276,95 @@ class LibvirtConnTestCase(test.NoDBTestCase):
                 break
         self.assertTrue(no_exist)
 
+    @mock.patch('nova.virt.libvirt.driver.LOG.warning')
+    @mock.patch.object(host.Host, 'has_min_version', return_value=True)
+    @mock.patch.object(host.Host, "get_capabilities")
+    def test_get_supported_perf_events_foo(self, mock_get_caps,
+                                           mock_min_version,
+                                           mock_warn):
+        self.flags(enabled_perf_events=['foo'], group='libvirt')
+
+        caps = vconfig.LibvirtConfigCaps()
+        caps.host = vconfig.LibvirtConfigCapsHost()
+        caps.host.cpu = vconfig.LibvirtConfigCPU()
+        caps.host.cpu.arch = "x86_64"
+        caps.host.topology = self._fake_caps_numa_topology()
+
+        mock_get_caps.return_value = caps
+        drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True)
+        events = drvr._get_supported_perf_events()
+
+        self.assertTrue(mock_warn.called)
+        self.assertEqual([], events)
+
+    @mock.patch.object(host.Host, "get_capabilities")
+    def _test_get_guest_with_perf(self, caps, events, mock_get_caps):
+        self.flags(enabled_perf_events=['cmt'], group='libvirt')
+        mock_get_caps.return_value = caps
+
+        drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True)
+        drvr.init_host('test_perf')
+        instance_ref = objects.Instance(**self.test_instance)
+        image_meta = objects.ImageMeta.from_dict(self.test_image_meta)
+
+        disk_info = blockinfo.get_disk_info(CONF.libvirt.virt_type,
+                                        instance_ref,
+                                        image_meta)
+        cfg = drvr._get_guest_config(instance_ref, [],
+                                 image_meta, disk_info)
+
+        self.assertEqual(events, cfg.perf_events)
+
+    @mock.patch.object(fakelibvirt, 'VIR_PERF_PARAM_CMT', True,
+                       create=True)
+    @mock.patch.object(host.Host, 'has_min_version', return_value=True)
+    def test_get_guest_with_perf_supported(self,
+                                 mock_min_version):
+        self.flags(enabled_perf_events=['cmt'], group='libvirt')
+        caps = vconfig.LibvirtConfigCaps()
+        caps.host = vconfig.LibvirtConfigCapsHost()
+        caps.host.cpu = vconfig.LibvirtConfigCPU()
+        caps.host.cpu.arch = "x86_64"
+        caps.host.topology = self._fake_caps_numa_topology()
+        feature = vconfig.LibvirtConfigGuestCPUFeature()
+        feature.name = 'cqm'
+        feature.policy = cpumodel.POLICY_REQUIRE
+        caps.host.cpu.features = set([feature])
+
+        self._test_get_guest_with_perf(caps, ['cmt'])
+
+    @mock.patch.object(host.Host, 'has_min_version')
+    def test_get_guest_with_perf_libvirt_unsupported(self, mock_min_version):
+
+        def fake_has_min_version(lv_ver=None, hv_ver=None, hv_type=None):
+            if lv_ver == libvirt_driver.MIN_LIBVIRT_PERF_VERSION:
+                return False
+            return True
+
+        mock_min_version.side_effect = fake_has_min_version
+        self.flags(enabled_perf_events=['cmt'], group='libvirt')
+
+        caps = vconfig.LibvirtConfigCaps()
+        caps.host = vconfig.LibvirtConfigCapsHost()
+        caps.host.cpu = vconfig.LibvirtConfigCPU()
+        caps.host.cpu.arch = "x86_64"
+
+        self._test_get_guest_with_perf(caps, [])
+
+    @mock.patch.object(fakelibvirt, 'VIR_PERF_PARAM_CMT', True,
+                       create=True)
+    @mock.patch.object(host.Host, 'has_min_version', return_value=True)
+    def test_get_guest_with_perf_host_unsupported(self,
+                                             mock_min_version):
+        self.flags(enabled_perf_events=['cmt'], group='libvirt')
+        caps = vconfig.LibvirtConfigCaps()
+        caps.host = vconfig.LibvirtConfigCapsHost()
+        caps.host.cpu = vconfig.LibvirtConfigCPU()
+        caps.host.cpu.arch = "x86_64"
+        caps.host.topology = self._fake_caps_numa_topology()
+
+        self._test_get_guest_with_perf(caps, [])
+
     def test_xml_and_uri_no_ramdisk_no_kernel(self):
         instance_data = dict(self.test_instance)
         self._check_xml_and_uri(instance_data,
@@ -10133,7 +10222,8 @@ class LibvirtConnTestCase(test.NoDBTestCase):
             mock.patch.object(drvr, "_do_quality_warnings",
                               return_value=None),
             mock.patch.object(objects.Service, "get_by_compute_host",
-                              return_value=service_mock)):
+                              return_value=service_mock),
+            mock.patch.object(host.Host, "get_capabilities")):
 
             drvr.init_host("wibble")
             self.assertRaises(exception.HypervisorUnavailable,
@@ -10155,7 +10245,8 @@ class LibvirtConnTestCase(test.NoDBTestCase):
             mock.patch.object(drvr, "_do_quality_warnings",
                               return_value=None),
             mock.patch.object(objects.Service, "get_by_compute_host",
-                              return_value=service_mock)):
+                              return_value=service_mock),
+            mock.patch.object(host.Host, "get_capabilities")):
 
             drvr.init_host("wibble")
             drvr.get_num_instances()

nova/virt/libvirt/config.py

@@ -2019,6 +2019,7 @@ class LibvirtConfigGuest(LibvirtConfigObject):
         self.devices = []
         self.metadata = []
         self.idmaps = []
+        self.perf_events = []
 
     def _format_basic_props(self, root):
         root.append(self._text_node("uuid", self.uuid))
@@ -2102,6 +2103,15 @@ class LibvirtConfigGuest(LibvirtConfigObject):
             idmaps.append(idmap.format_dom())
         root.append(idmaps)
 
+    def _format_perf_events(self, root):
+        if len(self.perf_events) == 0:
+            return
+        perfs = etree.Element("perf")
+        for pe in self.perf_events:
+            event = etree.Element("event", name=pe, enabled="yes")
+            perfs.append(event)
+        root.append(perfs)
+
     def format_dom(self):
         root = super(LibvirtConfigGuest, self).format_dom()
 
@@ -2128,6 +2138,8 @@ class LibvirtConfigGuest(LibvirtConfigObject):
 
         self._format_idmaps(root)
 
+        self._format_perf_events(root)
+
         return root
 
     def parse_dom(self, xmldoc):
@@ -2167,10 +2179,17 @@ class LibvirtConfigGuest(LibvirtConfigObject):
                 obj = LibvirtConfigGuestCPU()
                 obj.parse_dom(c)
                 self.cpu = obj
+            elif c.tag == 'perf':
+                for p in c.getchildren():
+                    if p.get('enabled') and p.get('enabled') == 'yes':
+                        self.add_perf_event(p.get('name'))
 
     def add_device(self, dev):
         self.devices.append(dev)
 
+    def add_perf_event(self, event):
+        self.perf_events.append(event)
+
     def set_clock(self, clk):
         self.clock = clk
 

nova/virt/libvirt/driver.py

@@ -297,6 +297,15 @@ MIN_QEMU_OTHER_ARCH = {arch.S390: MIN_QEMU_S390_VERSION,
                        arch.PPC64LE: MIN_QEMU_PPC64_VERSION,
                       }
 
+# perf events support
+MIN_LIBVIRT_PERF_VERSION = (1, 3, 3)
+LIBVIRT_PERF_EVENT_PREFIX = 'VIR_PERF_PARAM_'
+
+PERF_EVENTS_CPU_FLAG_MAPPING = {'cmt': 'cqm',
+                                'mbml': 'cqm_mbm_local',
+                                'mbmt': 'cqm_mbm_total',
+                               }
+
 
 class LibvirtDriver(driver.ComputeDriver):
     capabilities = {
@@ -320,6 +329,7 @@ class LibvirtDriver(driver.ComputeDriver):
         self._fc_wwnns = None
         self._fc_wwpns = None
         self._caps = None
+        self._supported_perf_events = []
         self.firewall_driver = firewall.load_driver(
             DEFAULT_FIREWALL_DRIVER,
             host=self._host)
@@ -442,6 +452,8 @@ class LibvirtDriver(driver.ComputeDriver):
 
         self._parse_migration_flags()
 
+        self._supported_perf_events = self._get_supported_perf_events()
+
         if (CONF.libvirt.virt_type == 'lxc' and
                 not (CONF.libvirt.uid_maps and CONF.libvirt.gid_maps)):
             LOG.warning(_LW("Running libvirt-lxc without user namespaces is "
@@ -4148,6 +4160,35 @@ class LibvirtDriver(driver.ComputeDriver):
                 self._host.has_min_version(MIN_LIBVIRT_UEFI_VERSION) and
                 os.path.exists(DEFAULT_UEFI_LOADER_PATH[caps.host.cpu.arch]))
 
+    def _get_supported_perf_events(self):
+
+        if (len(CONF.libvirt.enabled_perf_events) == 0 or
+             not self._host.has_min_version(MIN_LIBVIRT_PERF_VERSION)):
+            return []
+
+        supported_events = []
+        host_cpu_info = self._get_cpu_info()
+        for event in CONF.libvirt.enabled_perf_events:
+            if self._supported_perf_event(event, host_cpu_info['features']):
+                supported_events.append(event)
+        return supported_events
+
+    def _supported_perf_event(self, event, cpu_features):
+
+        libvirt_perf_event_name = LIBVIRT_PERF_EVENT_PREFIX + event.upper()
+
+        if not hasattr(libvirt, libvirt_perf_event_name):
+            LOG.warning(_LW("Libvirt doesn't support event type %s."),
+                        event)
+            return False
+
+        if (event in PERF_EVENTS_CPU_FLAG_MAPPING
+            and PERF_EVENTS_CPU_FLAG_MAPPING[event] not in cpu_features):
+            LOG.warning(_LW("Host does not support event type %s."), event)
+            return False
+
+        return True
+
     def _configure_guest_by_virt_type(self, guest, virt_type, caps, instance,
                                       image_meta, flavor, root_device_name):
         if virt_type == "xen":
@@ -4320,6 +4361,9 @@ class LibvirtDriver(driver.ComputeDriver):
                                                           instance))
         guest.idmaps = self._get_guest_idmaps()
 
+        for event in self._supported_perf_events:
+            guest.add_perf_event(event)
+
         self._update_guest_cputune(guest, flavor, virt_type)
 
         guest.cpu = self._get_guest_cpu_config(

releasenotes/notes/add-perf-event-e1385b6b6346fbda.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - Add perf event support for libvirt driver.
+    This can be done by adding new configure option
+    'enabled_perf_events' in libvirt section of
+    nova.conf. This feature requires libvirt>=1.3.3.