diff --git a/nova/conf/virt.py b/nova/conf/virt.py index 7574ca8b87b8..be485aa50a96 100644 --- a/nova/conf/virt.py +++ b/nova/conf/virt.py @@ -299,6 +299,29 @@ timeout_nbd = cfg.IntOpt( help='Amount of time, in seconds, to wait for NBD ' 'device start up.') +reserved_memory_pages = cfg.MultiStrOpt( + "reserved_memory_pages", + default=[], + help="""Reserves amount of huge pages per NUMA host cells + +Possible values: + +* A list of valid strings which reflect NUMA node ID, page size and + number of pages reserved separated by punctuation mark + colon. Default unit is KiB. + + reserved_memory_pages = ["0:2MB:64", "1:1GB:1"] + +Services which consume this: + +* nova-compute +* nova-scheduler + +Related options: + +* None""") + + ALL_OPTS = [vcpu_pin_set, compute_driver, default_ephemeral_format, @@ -312,7 +335,8 @@ ALL_OPTS = [vcpu_pin_set, injected_network_template, virt_mkfs, resize_fs_using_block_device, - timeout_nbd] + timeout_nbd, + reserved_memory_pages] def register_opts(conf): diff --git a/nova/exception.py b/nova/exception.py index 1d6077670920..d12b42d0b76d 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -2116,3 +2116,9 @@ class OsInfoNotFound(NotFound): class BuildRequestNotFound(NotFound): msg_fmt = _("BuildRequest not found for instance %(uuid)s") + + +class InvalidReservedMemoryPagesOption(Invalid): + msg_fmt = _("The format of the option 'reserved_memory_pages' is invalid. " + "(found '%(conf)s') Please refer to the nova " + "config-reference.") diff --git a/nova/tests/unit/virt/test_hardware.py b/nova/tests/unit/virt/test_hardware.py index fef3dd014143..bc367455730c 100644 --- a/nova/tests/unit/virt/test_hardware.py +++ b/nova/tests/unit/virt/test_hardware.py @@ -1349,6 +1349,71 @@ class NUMATopologyTest(test.NoDBTestCase): self.assertEqual(hostusage.cells[2].cpu_usage, 0) self.assertEqual(hostusage.cells[2].memory_usage, 0) + def test_topo_usage_reserved_page_size(self): + hosttopo = objects.NUMATopology(cells=[ + objects.NUMACell(id=0, cpuset=set([0, 1]), memory=512, + cpu_usage=0, memory_usage=0, mempages=[ + objects.NUMAPagesTopology( + size_kb=2048, + total=512, + used=128)], + siblings=[], pinned_cpus=set([])), + objects.NUMACell(id=1, cpuset=set([2, 3]), memory=512, + cpu_usage=0, memory_usage=0, mempages=[ + objects.NUMAPagesTopology( + size_kb=1048576, + total=5, + used=2)], + siblings=[], pinned_cpus=set([])), + ]) + instance1 = objects.InstanceNUMATopology(cells=[ + objects.InstanceNUMACell( + id=0, cpuset=set([0, 1]), memory=256, pagesize=2048), + objects.InstanceNUMACell( + id=1, cpuset=set([2, 3]), memory=1024, pagesize=1048576), + ]) + + # reserved_page_size is using a global variable so we prefer + # to create tests in a same context to avoid errors when + # running unit tests in parallel + + def test_success(): + hw.RESERVED_MEMORY_PAGES = None + self.flags(reserved_memory_pages=["0:2048:128", + "1:1048576:1"]) + hostusage = hw.numa_usage_from_instances( + hosttopo, [instance1]) + + self.assertEqual(hostusage.cells[0].mempages[0].size_kb, 2048) + self.assertEqual(hostusage.cells[0].mempages[0].total, 512) + # 128 reserved + 128 used by instance + self.assertEqual(hostusage.cells[0].mempages[0].used, 256) + + self.assertEqual(hostusage.cells[1].mempages[0].size_kb, 1048576) + self.assertEqual(hostusage.cells[1].mempages[0].total, 5) + # 1 reserved + 1 used by instance + 2 already used: + self.assertEqual(hostusage.cells[1].mempages[0].used, 4) + + def test_invalid_format(): + hw.RESERVED_MEMORY_PAGES = None + self.flags(reserved_memory_pages="1:2:3") + self.assertRaises( + exception.InvalidReservedMemoryPagesOption, + hw.numa_usage_from_instances, + hosttopo, [instance1]) + + def test_invalid_value(): + hw.RESERVED_MEMORY_PAGES = None + self.flags(reserved_memory_pages=["0:foo:bar"]) + self.assertRaises( + exception.InvalidReservedMemoryPagesOption, + hw.numa_usage_from_instances, + hosttopo, [instance1]) + + test_success() + test_invalid_format() + test_invalid_value() + def test_topo_usage_none(self): hosttopo = objects.NUMATopology(cells=[ objects.NUMACell(id=0, cpuset=set([0, 1]), memory=512, diff --git a/nova/virt/hardware.py b/nova/virt/hardware.py index 0ae1417798a5..47a2ba1379d8 100644 --- a/nova/virt/hardware.py +++ b/nova/virt/hardware.py @@ -38,6 +38,8 @@ MEMPAGES_SMALL = -1 MEMPAGES_LARGE = -2 MEMPAGES_ANY = -3 +RESERVED_MEMORY_PAGES = None + def get_vcpu_pin_set(): """Parsing vcpu_pin_set config. @@ -1248,6 +1250,36 @@ def numa_fit_instance_to_host( return objects.InstanceNUMATopology(cells=cells) +def _numa_reserved_memory_pages_from_cell(cell_id, pagesize): + global RESERVED_MEMORY_PAGES + try: + if CONF.reserved_memory_pages and not RESERVED_MEMORY_PAGES: + RESERVED_MEMORY_PAGES = collections.defaultdict(dict) + for cfg in CONF.reserved_memory_pages: + node, pagesize, reserved = cfg.split(":", 2) + node = int(node) + try: + pagesize = int(pagesize) + except ValueError: + pagesize = strutils.string_to_bytes( + pagesize, return_int=True) / units.Ki + RESERVED_MEMORY_PAGES[int(node)][pagesize] = int(reserved) + if RESERVED_MEMORY_PAGES: + return RESERVED_MEMORY_PAGES.get(cell_id, {}).get(pagesize, 0) + except ValueError: + raise exception.InvalidReservedMemoryPagesOption( + conf=CONF.reserved_memory_pages) + return 0 + + +def _numa_reserved_memory_pages(hostcells): + for hostcell in hostcells: + for pages in hostcell.mempages: + pages.used = ( + pages.used + _numa_reserved_memory_pages_from_cell( + hostcell.id, pages.size_kb)) + + def _numa_pagesize_usage_from_cell(hostcell, instancecell, sign): topo = [] for pages in hostcell.mempages: @@ -1326,6 +1358,9 @@ def numa_usage_from_instances(host, instances, free=False): cells.append(newcell) + # Compute reserved memory pages + _numa_reserved_memory_pages(cells) + return objects.NUMATopology(cells=cells) diff --git a/releasenotes/notes/reserved-hugepages-per-nodes-f36225d5fca807e4.yaml b/releasenotes/notes/reserved-hugepages-per-nodes-f36225d5fca807e4.yaml new file mode 100644 index 000000000000..f66fd4ffe49f --- /dev/null +++ b/releasenotes/notes/reserved-hugepages-per-nodes-f36225d5fca807e4.yaml @@ -0,0 +1,4 @@ +--- +features: + - Adds reserved_memory_pages option to reserve + amount of hugepages used by third part components. \ No newline at end of file