diff --git a/nova/conf/virt.py b/nova/conf/virt.py index 331f712f1159..aa6e0a250ef1 100644 --- a/nova/conf/virt.py +++ b/nova/conf/virt.py @@ -13,6 +13,7 @@ # under the License. from oslo_config import cfg +from oslo_config import types from nova.conf import paths @@ -348,6 +349,30 @@ Interdependencies to other options: configured as HVM. """) +reserved_huge_pages = cfg.MultiOpt( + "reserved_huge_pages", + item_type=types.Dict, + help="""Reserves a number of huge/large memory pages per NUMA host cells + +Possible values: + +* A list of valid key=value which reflect NUMA node ID, page size + (Default unit is KiB) and number of pages to be reserved. + + reserved_huge_pages = node=0,size=2048,count=64 + reserved_huge_pages = node=1,size=1GB,count=1 + + In this example we are reserving on NUMA node 0 64 pages of 2MiB + and on NUMA node 1 1 page of 1GiB. + +Services which consume this: + +* nova-compute + +Related options: + +* None""") + ALL_OPTS = [vcpu_pin_set, compute_driver, @@ -367,7 +392,8 @@ ALL_OPTS = [vcpu_pin_set, image_cache_subdirectory_name, remove_unused_base_images, remove_unused_original_minimum_age_seconds, - pointer_model] + pointer_model, + reserved_huge_pages] def register_opts(conf): diff --git a/nova/exception.py b/nova/exception.py index b9dee4a9bfa6..1219fb59e558 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -2137,3 +2137,9 @@ class BuildRequestNotFound(NotFound): class AttachInterfaceNotSupported(Invalid): msg_fmt = _("Attaching interfaces is not supported for " "instance %(instance)s.") + + +class InvalidReservedMemoryPagesOption(Invalid): + msg_fmt = _("The format of the option 'reserved_huge_pages' is invalid. " + "(found '%(conf)s') Please refer to the nova " + "config-reference.") diff --git a/nova/objects/numa.py b/nova/objects/numa.py index 8fdf43d31fb2..86ca18faa32d 100644 --- a/nova/objects/numa.py +++ b/nova/objects/numa.py @@ -13,6 +13,7 @@ # under the License. from oslo_serialization import jsonutils +from oslo_utils import versionutils from nova import exception from nova.objects import base @@ -148,14 +149,23 @@ class NUMACell(base.NovaObject): @base.NovaObjectRegistry.register class NUMAPagesTopology(base.NovaObject): # Version 1.0: Initial version - VERSION = '1.0' + # Version 1.1: Adds reserved field + VERSION = '1.1' fields = { 'size_kb': fields.IntegerField(), 'total': fields.IntegerField(), 'used': fields.IntegerField(default=0), + 'reserved': fields.IntegerField(default=0), } + def obj_make_compatible(self, primitive, target_version): + super(NUMAPagesTopology, self).obj_make_compatible(primitive, + target_version) + target_version = versionutils.convert_version_to_tuple(target_version) + if target_version < (1, 1): + primitive.pop('reserved', None) + def __eq__(self, other): return all_things_equal(self, other) @@ -165,7 +175,11 @@ class NUMAPagesTopology(base.NovaObject): @property def free(self): """Returns the number of avail pages.""" - return self.total - self.used + if not self.obj_attr_is_set('reserved'): + # In case where an old compute node is sharing resource to + # an updated node we must ensure that this property is defined. + self.reserved = 0 + return self.total - self.used - self.reserved @property def free_kb(self): diff --git a/nova/tests/unit/objects/test_numa.py b/nova/tests/unit/objects/test_numa.py index 64fa8662cf7a..61b7f4ec02bf 100644 --- a/nova/tests/unit/objects/test_numa.py +++ b/nova/tests/unit/objects/test_numa.py @@ -142,13 +142,20 @@ class _TestNUMA(object): objects.NUMAPagesTopology( size_kb=4, total=1548736, used=0), objects.NUMAPagesTopology( - size_kb=2048, total=513, used=0)]) # 1,002G + size_kb=2048, total=513, used=0), + objects.NUMAPagesTopology( + size_kb=1048576, total=4, used=1, reserved=1)]) pagesize = 2048 - self.assertTrue(cell.can_fit_hugepages(pagesize, 2 ** 20)) self.assertFalse(cell.can_fit_hugepages(pagesize, 2 ** 21)) self.assertFalse(cell.can_fit_hugepages(pagesize, 2 ** 19 + 1)) + + pagesize = 1048576 + self.assertTrue(cell.can_fit_hugepages(pagesize, 2 ** 20)) + self.assertTrue(cell.can_fit_hugepages(pagesize, 2 ** 20 * 2)) + self.assertFalse(cell.can_fit_hugepages(pagesize, 2 ** 20 * 3)) + self.assertRaises( exception.MemoryPageSizeNotSupported, cell.can_fit_hugepages, 12345, 2 ** 20) @@ -239,6 +246,13 @@ class _TestNUMA(object): mempages=[pt2]) self.assertNotEqual(cell1, cell2) + def test_reserved_property_not_set(self): + p = objects.NUMAPagesTopology( + # To have reserved not set is similar than to have receive + # a NUMAPageTopology version 1.0 + size_kb=1024, total=64, used=32) + self.assertEqual(32, p.free) + class TestNUMA(test_objects._LocalTest, _TestNUMA): diff --git a/nova/tests/unit/objects/test_objects.py b/nova/tests/unit/objects/test_objects.py index 682cd16b652c..d624686cbe69 100644 --- a/nova/tests/unit/objects/test_objects.py +++ b/nova/tests/unit/objects/test_objects.py @@ -1163,7 +1163,7 @@ object_data = { 'MonitorMetricList': '1.1-15ecf022a68ddbb8c2a6739cfc9f8f5e', 'NotificationPublisher': '1.0-bbbc1402fb0e443a3eb227cc52b61545', 'NUMACell': '1.2-74fc993ac5c83005e76e34e8487f1c05', - 'NUMAPagesTopology': '1.0-c71d86317283266dc8364c149155e48e', + 'NUMAPagesTopology': '1.1-edab9fa2dc43c117a38d600be54b4542', 'NUMATopology': '1.2-c63fad38be73b6afd04715c9c1b29220', 'NUMATopologyLimits': '1.0-9463e0edd40f64765ae518a539b9dfd2', 'Network': '1.2-a977ab383aa462a479b2fae8211a5dde', diff --git a/nova/tests/unit/virt/test_hardware.py b/nova/tests/unit/virt/test_hardware.py index d1e49647562e..9d85366dd1ed 100644 --- a/nova/tests/unit/virt/test_hardware.py +++ b/nova/tests/unit/virt/test_hardware.py @@ -1367,6 +1367,79 @@ class NUMATopologyTest(test.NoDBTestCase): self.assertEqual(hostusage.cells[2].cpu_usage, 0) self.assertEqual(hostusage.cells[2].memory_usage, 0) + def _topo_usage_reserved_page_size(self): + reserved = hw.numa_get_reserved_huge_pages() + hosttopo = objects.NUMATopology(cells=[ + objects.NUMACell(id=0, cpuset=set([0, 1]), memory=512, + cpu_usage=0, memory_usage=0, mempages=[ + objects.NUMAPagesTopology( + size_kb=2048, + total=512, + used=128, + reserved=reserved[0][2048])], + siblings=[], pinned_cpus=set([])), + objects.NUMACell(id=1, cpuset=set([2, 3]), memory=512, + cpu_usage=0, memory_usage=0, mempages=[ + objects.NUMAPagesTopology( + size_kb=1048576, + total=5, + used=2, + reserved=reserved[1][1048576])], + siblings=[], pinned_cpus=set([])), + ]) + instance1 = objects.InstanceNUMATopology(cells=[ + objects.InstanceNUMACell( + id=0, cpuset=set([0, 1]), memory=256, pagesize=2048), + objects.InstanceNUMACell( + id=1, cpuset=set([2, 3]), memory=1024, pagesize=1048576), + ]) + return hosttopo, instance1 + + def test_numa_get_reserved_huge_pages(self): + reserved = hw.numa_get_reserved_huge_pages() + self.assertEqual({}, reserved) + self.flags(reserved_huge_pages=[ + {'node': 3, 'size': 2048, 'count': 128}, + {'node': 3, 'size': '1GB', 'count': 4}, + {'node': 6, 'size': '2MB', 'count': 64}, + {'node': 9, 'size': '1GB', 'count': 1}]) + reserved = hw.numa_get_reserved_huge_pages() + self.assertEqual({2048: 128, 1048576: 4}, reserved[3]) + self.assertEqual({2048: 64}, reserved[6]) + self.assertEqual({1048576: 1}, reserved[9]) + + def test_reserved_hugepgaes_success(self): + self.flags(reserved_huge_pages=[ + {'node': 0, 'size': 2048, 'count': 128}, + {'node': 1, 'size': 1048576, 'count': 1}]) + hosttopo, instance1 = self._topo_usage_reserved_page_size() + hostusage = hw.numa_usage_from_instances( + hosttopo, [instance1]) + + self.assertEqual(hostusage.cells[0].mempages[0].size_kb, 2048) + self.assertEqual(hostusage.cells[0].mempages[0].total, 512) + self.assertEqual(hostusage.cells[0].mempages[0].used, 256) + # 128 already used + 128 used by instance + 128 reserved + self.assertEqual(hostusage.cells[0].mempages[0].free, 128) + + self.assertEqual(hostusage.cells[1].mempages[0].size_kb, 1048576) + self.assertEqual(hostusage.cells[1].mempages[0].total, 5) + self.assertEqual(hostusage.cells[1].mempages[0].used, 3) + # 2 already used + 1 used by instance + 1 reserved + self.assertEqual(hostusage.cells[1].mempages[0].free, 1) + + def test_reserved_huge_pages_invalid_format(self): + self.flags(reserved_huge_pages=[{'node': 0, 'size': 2048}]) + self.assertRaises( + exception.InvalidReservedMemoryPagesOption, + self._topo_usage_reserved_page_size) + + def test_reserved_huge_pages_invalid_value(self): + self.flags(reserved_huge_pages=["0:foo:bar"]) + self.assertRaises( + exception.InvalidReservedMemoryPagesOption, + self._topo_usage_reserved_page_size) + def test_topo_usage_none(self): hosttopo = objects.NUMATopology(cells=[ objects.NUMACell(id=0, cpuset=set([0, 1]), memory=512, diff --git a/nova/virt/hardware.py b/nova/virt/hardware.py index 322af2022c34..376ce79067b3 100644 --- a/nova/virt/hardware.py +++ b/nova/virt/hardware.py @@ -1249,6 +1249,36 @@ def numa_fit_instance_to_host( return objects.InstanceNUMATopology(cells=cells) +def numa_get_reserved_huge_pages(): + """Returns reserved memory pages from host option + + Based from the compute node option reserved_huge_pages, this + method will return a well formatted list of dict which can be used + to build NUMATopology. + + :raises: exceptionInvalidReservedMemoryPagesOption is option is + not corretly set. + + :returns: a list of dict ordered by NUMA node ids; keys of dict + are pages size where values are the number reserved. + """ + bucket = {} + if CONF.reserved_huge_pages: + try: + bucket = collections.defaultdict(dict) + for cfg in CONF.reserved_huge_pages: + try: + pagesize = int(cfg['size']) + except ValueError: + pagesize = strutils.string_to_bytes( + cfg['size'], return_int=True) / units.Ki + bucket[int(cfg['node'])][pagesize] = int(cfg['count']) + except (ValueError, TypeError, KeyError): + raise exception.InvalidReservedMemoryPagesOption( + conf=CONF.reserved_huge_pages) + return bucket + + def _numa_pagesize_usage_from_cell(hostcell, instancecell, sign): topo = [] for pages in hostcell.mempages: @@ -1258,7 +1288,8 @@ def _numa_pagesize_usage_from_cell(hostcell, instancecell, sign): total=pages.total, used=max(0, pages.used + instancecell.memory * units.Ki / - pages.size_kb * sign))) + pages.size_kb * sign), + reserved=pages.reserved if 'reserved' in pages else 0)) else: topo.append(pages) return topo diff --git a/releasenotes/notes/reserved-hugepages-per-nodes-f36225d5fca807e4.yaml b/releasenotes/notes/reserved-hugepages-per-nodes-f36225d5fca807e4.yaml index d780c7e98e22..c32a7a988bda 100644 --- a/releasenotes/notes/reserved-hugepages-per-nodes-f36225d5fca807e4.yaml +++ b/releasenotes/notes/reserved-hugepages-per-nodes-f36225d5fca807e4.yaml @@ -1,2 +1,3 @@ ---- -prelude: > +features: + - Adds reserved_huge_pages option to reserve + amount of huge pages used by third party components. \ No newline at end of file