From 3e9cbe903bad635ea96421fa8fb054ecbe8ed13b Mon Sep 17 00:00:00 2001
From: Sahid Orentino Ferdjaoui <sahid.ferdjaoui@redhat.com>
Date: Tue, 22 Nov 2016 11:52:25 -0500
Subject: [PATCH] libvirt: make emulator threads to run on the reserved pCPU

In this commit we finally make libvirt to use the pCPU reserved to pin
emulator threads.

implements blueprint libvirt-emulator-threads-policy

Change-Id: I25baf488b1c8a3959bdd6abe3ce3c1f6429b6eab
---
 nova/tests/unit/virt/libvirt/test_driver.py | 56 +++++++++++++++++++++
 nova/virt/libvirt/driver.py                 | 10 +++-
 2 files changed, 65 insertions(+), 1 deletion(-)

diff --git a/nova/tests/unit/virt/libvirt/test_driver.py b/nova/tests/unit/virt/libvirt/test_driver.py
index 6a9f52a4e9e8..849fb23efd03 100644
--- a/nova/tests/unit/virt/libvirt/test_driver.py
+++ b/nova/tests/unit/virt/libvirt/test_driver.py
@@ -2886,6 +2886,62 @@ class LibvirtConnTestCase(test.NoDBTestCase):
             # which are 6, 7
             self.assertEqual(set([2, 3]), cfg.cputune.vcpusched[0].vcpus)
 
+    def test_get_guest_config_numa_host_instance_isolated_emulator_threads(
+            self):
+        instance_topology = objects.InstanceNUMATopology(
+            emulator_threads_policy=(
+                fields.CPUEmulatorThreadsPolicy.ISOLATE),
+            cells=[
+                objects.InstanceNUMACell(
+                    id=0, cpuset=set([0, 1]),
+                    memory=1024, pagesize=2048,
+                    cpu_policy=fields.CPUAllocationPolicy.DEDICATED,
+                    cpu_pinning={0: 4, 1: 5},
+                    cpuset_reserved=set([6])),
+                objects.InstanceNUMACell(
+                    id=1, cpuset=set([2, 3]),
+                    memory=1024, pagesize=2048,
+                    cpu_policy=fields.CPUAllocationPolicy.DEDICATED,
+                    cpu_pinning={2: 7, 3: 8},
+                    cpuset_reserved=set([]))])
+
+        instance_ref = objects.Instance(**self.test_instance)
+        instance_ref.numa_topology = instance_topology
+        image_meta = objects.ImageMeta.from_dict(self.test_image_meta)
+
+        caps = vconfig.LibvirtConfigCaps()
+        caps.host = vconfig.LibvirtConfigCapsHost()
+        caps.host.cpu = vconfig.LibvirtConfigCPU()
+        caps.host.cpu.arch = "x86_64"
+        caps.host.topology = fakelibvirt.NUMATopology()
+
+        drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True)
+        disk_info = blockinfo.get_disk_info(CONF.libvirt.virt_type,
+                                            instance_ref, image_meta)
+
+        with test.nested(
+                mock.patch.object(
+                    objects.InstanceNUMATopology, "get_by_instance_uuid",
+                    return_value=instance_topology),
+                mock.patch.object(host.Host, 'has_min_version',
+                                  return_value=True),
+                mock.patch.object(host.Host, "get_capabilities",
+                                  return_value=caps),
+                mock.patch.object(
+                    hardware, 'get_vcpu_pin_set',
+                    return_value=set([4, 5, 6, 7, 8])),
+                mock.patch.object(host.Host, 'get_online_cpus',
+                                  return_value=set(range(10))),
+                ):
+            cfg = drvr._get_guest_config(instance_ref, [],
+                                         image_meta, disk_info)
+
+            self.assertEqual(set([6]), cfg.cputune.emulatorpin.cpuset)
+            self.assertEqual(set([4]), cfg.cputune.vcpupin[0].cpuset)
+            self.assertEqual(set([5]), cfg.cputune.vcpupin[1].cpuset)
+            self.assertEqual(set([7]), cfg.cputune.vcpupin[2].cpuset)
+            self.assertEqual(set([8]), cfg.cputune.vcpupin[3].cpuset)
+
     def test_get_cpu_numa_config_from_instance(self):
         topology = objects.InstanceNUMATopology(cells=[
             objects.InstanceNUMACell(id=0, cpuset=set([1, 2]), memory=128),
diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py
index 3544d9aa247d..a2be1919ea46 100644
--- a/nova/virt/libvirt/driver.py
+++ b/nova/virt/libvirt/driver.py
@@ -3978,6 +3978,9 @@ class LibvirtDriver(driver.ComputeDriver):
                 numa_memnodes = [vconfig.LibvirtConfigGuestNUMATuneMemNode()
                                  for _ in guest_cpu_numa_config.cells]
 
+                emulator_threads_isolated = (
+                    instance_numa_topology.emulator_threads_isolated)
+
                 vcpus_rt = set([])
                 wants_realtime = hardware.is_realtime_enabled(flavor)
                 if wants_realtime:
@@ -3994,6 +3997,8 @@ class LibvirtDriver(driver.ComputeDriver):
                         CONF.libvirt.realtime_scheduler_priority)
                     guest_cpu_tune.vcpusched.append(vcpusched)
 
+                # TODO(sahid): Defining domain topology should be
+                # refactored.
                 for host_cell in topology.cells:
                     for guest_node_id, guest_config_cell in enumerate(
                             guest_cpu_numa_config.cells):
@@ -4022,7 +4027,10 @@ class LibvirtDriver(driver.ComputeDriver):
                                     pin_cpuset.cpuset = set([pcpu])
                                 else:
                                     pin_cpuset.cpuset = host_cell.cpuset
-                                if not wants_realtime or cpu not in vcpus_rt:
+                                if emulator_threads_isolated:
+                                    emupcpus.extend(
+                                        object_numa_cell.cpuset_reserved)
+                                elif not wants_realtime or cpu not in vcpus_rt:
                                     # - If realtime IS NOT enabled, the
                                     #   emulator threads are allowed to float
                                     #   across all the pCPUs associated with