From b11dbfa4902cdd74bad3745db177d80b1c8b07c6 Mon Sep 17 00:00:00 2001 From: Nikola Dipanov Date: Fri, 5 Dec 2014 13:52:47 +0100 Subject: [PATCH] Make numa_usage_from_instances consider CPU pinning If instance exposes any CPU pinning information, it will be taken into consideration when calculating NUMA cell usage of the host. We also add some convenience methods to NUMACell object for pinning and unpinning CPUs, that will raise an error if the data is inconsistent, and a wrapper for checking weather CPU pinning was actually requested (to avoid an error prone 'is not None' in places where the check is needed). Change-Id: I43543c526d3733572f4185fbc9aa2207e9b2154c Blueprint: virt-driver-cpu-pinning --- nova/exception.py | 5 ++ nova/objects/instance_numa_topology.py | 8 +++ nova/objects/numa.py | 12 +++++ .../objects/test_instance_numa_topology.py | 15 ++++++ nova/tests/unit/objects/test_numa.py | 17 +++++- nova/tests/unit/virt/test_hardware.py | 53 +++++++++++++++++++ nova/virt/hardware.py | 25 ++++++--- 7 files changed, 126 insertions(+), 9 deletions(-) diff --git a/nova/exception.py b/nova/exception.py index d08758884db5..7cc0afd0bb06 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -1822,3 +1822,8 @@ class MemoryPageSizeForbidden(Invalid): class MemoryPageSizeNotSupported(Invalid): msg_fmt = _("Page size %(pagesize)s is not supported by the host.") + + +class CPUPinningInvalid(Invalid): + msg_fmt = _("Cannot pin/unpin cpus %(requested)s from the following " + "pinned set %(pinned)s") diff --git a/nova/objects/instance_numa_topology.py b/nova/objects/instance_numa_topology.py index 582d550fdea8..cb1284ffc586 100644 --- a/nova/objects/instance_numa_topology.py +++ b/nova/objects/instance_numa_topology.py @@ -92,6 +92,10 @@ class InstanceNUMACell(base.NovaObject, return map(set, zip(*[iter(cpu_list)] * threads)) + @property + def cpu_pinning_requested(self): + return self.cpu_pinning is not None + def pin(self, vcpu, pcpu): if vcpu not in self.cpuset: return @@ -202,3 +206,7 @@ class InstanceNUMATopology(base.NovaObject, return cls(cells=[ InstanceNUMACell._from_dict(cell_dict) for cell_dict in data_dict.get('cells', [])]) + + @property + def cpu_pinning_requested(self): + return all(cell.cpu_pinning_requested for cell in self.cells) diff --git a/nova/objects/numa.py b/nova/objects/numa.py index 21936f5efd39..2c628878f084 100644 --- a/nova/objects/numa.py +++ b/nova/objects/numa.py @@ -67,6 +67,18 @@ class NUMACell(base.NovaObject, def avail_memory(self): return self.memory - self.memory_usage + def pin_cpus(self, cpus): + if self.pinned_cpus & cpus: + raise exception.CPUPinningInvalid(requested=list(cpus), + pinned=list(self.pinned_cpus)) + self.pinned_cpus |= cpus + + def unpin_cpus(self, cpus): + if (self.pinned_cpus & cpus) != cpus: + raise exception.CPUPinningInvalid(requested=list(cpus), + pinned=list(self.pinned_cpus)) + self.pinned_cpus -= cpus + def _to_dict(self): return { 'id': self.id, diff --git a/nova/tests/unit/objects/test_instance_numa_topology.py b/nova/tests/unit/objects/test_instance_numa_topology.py index 84a003fb6bec..f3b73f2e71b9 100644 --- a/nova/tests/unit/objects/test_instance_numa_topology.py +++ b/nova/tests/unit/objects/test_instance_numa_topology.py @@ -10,6 +10,7 @@ # License for the specific language governing permissions and limitations # under the License. +import copy import uuid import mock @@ -140,6 +141,20 @@ class _TestInstanceNUMATopology(object): inst_cell = objects.InstanceNUMACell() self.assertEqual(0, len(inst_cell.obj_get_changes())) + def test_cpu_pinning_requested_cell(self): + inst_cell = objects.InstanceNUMACell(cpuset=set([0, 1, 2, 3]), + cpu_pinning=None) + self.assertFalse(inst_cell.cpu_pinning_requested) + inst_cell.cpu_pinning = {} + self.assertTrue(inst_cell.cpu_pinning_requested) + + def test_cpu_pinning_requested(self): + fake_topo_obj = copy.deepcopy(fake_obj_numa_topology) + self.assertFalse(fake_topo_obj.cpu_pinning_requested) + for cell in fake_topo_obj.cells: + cell.cpu_pinning = dict(zip(*map(list, [cell.cpuset] * 2))) + self.assertTrue(fake_topo_obj.cpu_pinning_requested) + class TestInstanceNUMATopology(test_objects._LocalTest, _TestInstanceNUMATopology): diff --git a/nova/tests/unit/objects/test_numa.py b/nova/tests/unit/objects/test_numa.py index 88000827ef59..8ba245a6c7b1 100644 --- a/nova/tests/unit/objects/test_numa.py +++ b/nova/tests/unit/objects/test_numa.py @@ -33,7 +33,7 @@ class _TestNUMA(object): self.assertEqual(d1, d2) - def test_pinning_logic(self): + def test_free_cpus(self): obj = objects.NUMATopology(cells=[ objects.NUMACell( id=0, cpuset=set([1, 2]), memory=512, @@ -48,6 +48,21 @@ class _TestNUMA(object): self.assertEqual(set([2]), obj.cells[0].free_cpus) self.assertEqual(set([3, 4]), obj.cells[1].free_cpus) + def test_pinning_logic(self): + numacell = objects.NUMACell(id=0, cpuset=set([1, 2, 3, 4]), memory=512, + cpu_usage=2, memory_usage=256, + pinned_cpus=set([1])) + numacell.pin_cpus(set([2, 3])) + self.assertEqual(set([4]), numacell.free_cpus) + self.assertRaises(exception.CPUPinningInvalid, + numacell.pin_cpus, set([1, 4])) + self.assertRaises(exception.CPUPinningInvalid, + numacell.pin_cpus, set([1, 6])) + self.assertRaises(exception.CPUPinningInvalid, + numacell.unpin_cpus, set([1, 4])) + numacell.unpin_cpus(set([1, 2, 3])) + self.assertEqual(set([1, 2, 3, 4]), numacell.free_cpus) + def test_pages_topology_wipe(self): pages_topology = objects.NUMAPagesTopology( size_kb=2048, total=1024, used=512) diff --git a/nova/tests/unit/virt/test_hardware.py b/nova/tests/unit/virt/test_hardware.py index 0ad35d3d7d87..1516a4b75594 100644 --- a/nova/tests/unit/virt/test_hardware.py +++ b/nova/tests/unit/virt/test_hardware.py @@ -1828,3 +1828,56 @@ class CPUPinningTestCase(test.NoDBTestCase, _CPUPinningTestCaseBase): memory=1024, cpu_pinning={})]) inst_topo = hw.numa_fit_instance_to_host(host_topo, inst_topo) self.assertIsNone(inst_topo) + + def test_cpu_pinning_usage_from_instances(self): + host_pin = objects.NUMATopology( + cells=[objects.NUMACell(id=0, cpuset=set([0, 1, 2, 3]), + memory=4096, cpu_usage=0, + memory_usage=0)]) + inst_pin_1 = objects.InstanceNUMATopology( + cells=[objects.InstanceNUMACell( + cpuset=set([0, 1]), id=0, cpu_pinning={0: 0, 1: 3}, + memory=2048)]) + inst_pin_2 = objects.InstanceNUMATopology( + cells = [objects.InstanceNUMACell( + cpuset=set([0, 1]), id=0, cpu_pinning={0: 1, 1: 2}, + memory=2048)]) + + host_pin = hw.numa_usage_from_instances( + host_pin, [inst_pin_1, inst_pin_2]) + self.assertEqual(set([0, 1, 2, 3]), + host_pin.cells[0].pinned_cpus) + + def test_cpu_pinning_usage_from_instances_free(self): + host_pin = objects.NUMATopology( + cells=[objects.NUMACell(id=0, cpuset=set([0, 1, 2, 3]), + memory=4096, cpu_usage=0, memory_usage=0, + pinned_cpus=set([0, 1, 3]))]) + inst_pin_1 = objects.InstanceNUMATopology( + cells=[objects.InstanceNUMACell( + cpuset=set([0]), memory=1024, cpu_pinning={0: 1}, id=0)]) + inst_pin_2 = objects.InstanceNUMATopology( + cells=[objects.InstanceNUMACell( + cpuset=set([0, 1]), memory=1024, id=0, + cpu_pinning={0: 0, 1: 3})]) + host_pin = hw.numa_usage_from_instances( + host_pin, [inst_pin_1, inst_pin_2], free=True) + self.assertEqual(set(), host_pin.cells[0].pinned_cpus) + + def test_host_usage_from_instances_fail(self): + host_pin = objects.NUMATopology( + cells=[objects.NUMACell(id=0, cpuset=set([0, 1, 2, 3]), + memory=4096, cpu_usage=0, + memory_usage=0)]) + inst_pin_1 = objects.InstanceNUMATopology( + cells=[objects.InstanceNUMACell( + cpuset=set([0, 1]), memory=2048, id=0, + cpu_pinning={0: 0, 1: 3})]) + inst_pin_2 = objects.InstanceNUMATopology( + cells = [objects.InstanceNUMACell( + cpuset=set([0, 1]), id=0, memory=2048, + cpu_pinning={0: 0, 1: 2})]) + + self.assertRaises(exception.CPUPinningInvalid, + hw.numa_usage_from_instances, host_pin, + [inst_pin_1, inst_pin_2]) diff --git a/nova/virt/hardware.py b/nova/virt/hardware.py index 4f549e1c190e..38807dc4a60f 100644 --- a/nova/virt/hardware.py +++ b/nova/virt/hardware.py @@ -779,7 +779,7 @@ def _numa_fit_instance_cell(host_cell, instance_cell, limit_cell=None): len(instance_cell.cpuset) > len(host_cell.cpuset)): return None - if instance_cell.cpu_pinning is not None: + if instance_cell.cpu_pinning_requested: new_instance_cell = _numa_fit_instance_cell_with_pinning( host_cell, instance_cell) if not new_instance_cell: @@ -1121,7 +1121,12 @@ def numa_usage_from_instances(host, instances, free=False): for hostcell in host.cells: memory_usage = hostcell.memory_usage cpu_usage = hostcell.cpu_usage - mempages = hostcell.mempages + + newcell = objects.NUMACell( + id=hostcell.id, cpuset=hostcell.cpuset, memory=hostcell.memory, + cpu_usage=0, memory_usage=0, + pinned_cpus=hostcell.pinned_cpus, siblings=hostcell.siblings) + for instance in instances: for instancecell in instance.cells: if instancecell.id == hostcell.id: @@ -1129,15 +1134,19 @@ def numa_usage_from_instances(host, instances, free=False): memory_usage + sign * instancecell.memory) cpu_usage = cpu_usage + sign * len(instancecell.cpuset) if instancecell.pagesize and instancecell.pagesize > 0: - mempages = _numa_pagesize_usage_from_cell( + newcell.mempages = _numa_pagesize_usage_from_cell( hostcell, instancecell, sign) + if instance.cpu_pinning_requested: + pinned_cpus = set(instancecell.cpu_pinning.values()) + if free: + newcell.unpin_cpus(pinned_cpus) + else: + newcell.pin_cpus(pinned_cpus) - cell = objects.NUMACell( - id=hostcell.id, cpuset=hostcell.cpuset, memory=hostcell.memory, - cpu_usage=max(0, cpu_usage), memory_usage=max(0, memory_usage), - mempages=mempages) + newcell.cpu_usage = max(0, cpu_usage) + newcell.memory_usage = max(0, memory_usage) - cells.append(cell) + cells.append(newcell) return objects.NUMATopology(cells=cells)