From 3f7cc63d942bbb7cbda3aae9718fa21f61deecf0 Mon Sep 17 00:00:00 2001 From: yatinkarel Date: Wed, 26 Apr 2023 13:41:43 +0000 Subject: [PATCH] Add config option to configure TB cache size Qemu>=5.0.0 bumped the default tb-cache size to 1GiB(from 32MiB) and this made it difficult to run multiple guest VMs on systems running with lower memory. With Libvirt>=8.0.0 it's possible to configure lower tb-cache size. Below config option is introduced to allow configure TB cache size as per environment needs, this only applies to 'virt_type=qemu':- [libvirt]tb_cache_size Also enable this flag in nova-next job. [1] https://github.com/qemu/qemu/commit/600e17b26 [2] https://gitlab.com/libvirt/libvirt/-/commit/58bf03f85 Closes-Bug: #1949606 Implements: blueprint libvirt-tb-cache-size Change-Id: I49d2276ff3d3cc5d560a1bd96f13408e798b256a --- .zuul.yaml | 3 ++ nova/conf/libvirt.py | 16 ++++++++ nova/tests/unit/virt/libvirt/test_config.py | 8 ++++ nova/tests/unit/virt/libvirt/test_driver.py | 41 +++++++++++++++++++ nova/virt/libvirt/config.py | 15 +++++++ nova/virt/libvirt/driver.py | 26 ++++++++++++ ...ibvirt-tb-cache-size-c0d0d672c0950393.yaml | 12 ++++++ 7 files changed, 121 insertions(+) create mode 100644 releasenotes/notes/libvirt-tb-cache-size-c0d0d672c0950393.yaml diff --git a/.zuul.yaml b/.zuul.yaml index e660431b002f..209e2a0b564d 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -362,6 +362,9 @@ # machine type attaches more devices by default than pc num_pcie_ports: 24 hw_machine_type: "x86_64=q35" + # Use lower TB cache than default(1GiB), only applicable with + # libvirt>=8.0.0 + tb_cache_size: 128 compute: # Switch off the provider association refresh, which should # reduce the number of placement calls in steady state. Added in diff --git a/nova/conf/libvirt.py b/nova/conf/libvirt.py index 204fe5c4b867..8b1c148dbd2b 100644 --- a/nova/conf/libvirt.py +++ b/nova/conf/libvirt.py @@ -118,6 +118,7 @@ Related options: * ``disk_prefix``: depends on this * ``cpu_mode``: depends on this * ``cpu_models``: depends on this +* ``tb_cache_size``: depends on this """), cfg.StrOpt('connection_uri', default='', @@ -900,6 +901,21 @@ Related options: * :oslo.config:option:`libvirt.device_detach_attempts` +"""), + cfg.IntOpt('tb_cache_size', + min=0, + help=""" +Qemu>=5.0.0 bumped the default tb-cache size to 1GiB(from 32MiB) and this +made it difficult to run multiple guest VMs on systems running with lower +memory. With Libvirt>=8.0.0 this config option can be used to configure +lower tb-cache size. + +Set it to > 0 to configure tb-cache for guest VMs. + +Related options: + +* ``compute_driver`` (libvirt) +* ``virt_type`` (qemu) """), ] diff --git a/nova/tests/unit/virt/libvirt/test_config.py b/nova/tests/unit/virt/libvirt/test_config.py index 3d0b5ae6853d..7402a09647e7 100644 --- a/nova/tests/unit/virt/libvirt/test_config.py +++ b/nova/tests/unit/virt/libvirt/test_config.py @@ -2414,6 +2414,14 @@ class LibvirtConfigGuestFeatureTest(LibvirtConfigBaseTest): xml = obj.to_xml() self.assertXmlEqual(xml, "") + def test_feature_tcg(self): + obj = config.LibvirtConfigGuestFeatureTCG(10) + obj.driver = "libvirt" + + xml = obj.to_xml() + self.assertXmlEqual(xml, '10' + '') + class LibvirtConfigGuestTest(LibvirtConfigBaseTest): diff --git a/nova/tests/unit/virt/libvirt/test_driver.py b/nova/tests/unit/virt/libvirt/test_driver.py index 774498b69f44..1cad45875bf4 100644 --- a/nova/tests/unit/virt/libvirt/test_driver.py +++ b/nova/tests/unit/virt/libvirt/test_driver.py @@ -21367,6 +21367,47 @@ class LibvirtConnTestCase(test.NoDBTestCase, cpu_model = drvr._get_cpu_model_mapping(expect_model[0]) self.assertEqual(cpu_model, expect_model[0]) + @mock.patch.object(fakelibvirt.Connection, 'getLibVersion', + return_value=versionutils.convert_version_to_int( + libvirt_driver.MIN_LIBVIRT_TB_CACHE_SIZE) - 1) + def test_supports_tb_cache_size_fail(self, mock_getversion): + self.flags(virt_type='qemu', group='libvirt') + self.flags(tb_cache_size=10, group='libvirt') + driver = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False) + + self.assertRaises(exception.InvalidConfiguration, + driver.init_host, 'dummyhost') + + @mock.patch.object(libvirt_driver.LibvirtDriver, + '_register_all_undefined_instance_details', + new=mock.Mock()) + @mock.patch.object(fakelibvirt.Connection, 'getLibVersion', + return_value=versionutils.convert_version_to_int( + libvirt_driver.MIN_LIBVIRT_TB_CACHE_SIZE)) + def test_supports_tb_cache_size_ok(self, mock_getversion): + self.flags(virt_type='qemu', group='libvirt') + self.flags(tb_cache_size=10, group='libvirt') + driver = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False) + driver.init_host('dummyhost') + + @mock.patch.object(fakelibvirt.Connection, 'getLibVersion', + return_value=versionutils.convert_version_to_int( + libvirt_driver.MIN_LIBVIRT_TB_CACHE_SIZE)) + def test_get_guest_config_feature_tcg(self, mock_getversion): + self.flags(virt_type='qemu', group='libvirt') + self.flags(tb_cache_size=10, group='libvirt') + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) + instance_ref = objects.Instance(**self.test_instance) + image_meta = objects.ImageMeta.from_dict(self.test_image_meta) + disk_info = blockinfo.get_disk_info( + CONF.libvirt.virt_type, instance_ref, image_meta) + + cfg = drvr._get_guest_config( + instance_ref, _fake_network_info(self), image_meta, disk_info) + + expected = '10' + self.assertXmlEqual(expected, cfg.features[2].to_xml()) + class TestGuestConfigSysinfoSerialOS(test.NoDBTestCase): def setUp(self): diff --git a/nova/virt/libvirt/config.py b/nova/virt/libvirt/config.py index 231283b8ddb6..2a1e703d31d2 100644 --- a/nova/virt/libvirt/config.py +++ b/nova/virt/libvirt/config.py @@ -2761,6 +2761,21 @@ class LibvirtConfigGuestFeatureSMM(LibvirtConfigGuestFeature): return root +class LibvirtConfigGuestFeatureTCG(LibvirtConfigGuestFeature): + + def __init__(self, cache_size, **kwargs): + super(LibvirtConfigGuestFeatureTCG, self).__init__("tcg", **kwargs) + + self.cache_size = str(cache_size) + + def format_dom(self): + root = super(LibvirtConfigGuestFeatureTCG, self).format_dom() + root.append(self._text_node("tb-cache", self.cache_size, + unit="MiB")) + + return root + + class LibvirtConfigGuestFeaturePMU(LibvirtConfigGuestFeature): def __init__(self, state, **kwargs): diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index ff70c9b74416..0621d91b8ddd 100644 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -229,6 +229,8 @@ MIN_LIBVIRT_VIOMMU_VIRTIO_MODEL = (8, 3, 0) MIN_LIBVIRT_AARCH64_CPU_COMPARE = (6, 9, 0) +MIN_LIBVIRT_TB_CACHE_SIZE = (8, 0, 0) + # Virtuozzo driver support MIN_VIRTUOZZO_VERSION = (7, 0, 0) @@ -743,6 +745,9 @@ class LibvirtDriver(driver.ComputeDriver): self._check_my_ip() + # TODO(ykarel) This can be dropped when MIN_LIBVIRT_VERSION>=8.0.0 + self._supports_tb_cache_size() + if (CONF.libvirt.virt_type == 'lxc' and not (CONF.libvirt.uid_maps and CONF.libvirt.gid_maps)): LOG.warning("Running libvirt-lxc without user namespaces is " @@ -1217,6 +1222,19 @@ class LibvirtDriver(driver.ComputeDriver): "cpu_shared_set' and '[compute] cpu_dedicated_set', " "respectively, and undefine 'vcpu_pin_set'.") + def _supports_tb_cache_size(self): + if ( + CONF.libvirt.virt_type == 'qemu' and + CONF.libvirt.tb_cache_size and + CONF.libvirt.tb_cache_size > 0 + ): + if not self._host.has_min_version(MIN_LIBVIRT_TB_CACHE_SIZE): + raise exception.InvalidConfiguration( + _("Nova requires libvirt version %s or greater " + "with '[libvirt] tb_cache_size' " + "configured.") % + libvirt_utils.version_to_string(MIN_LIBVIRT_TB_CACHE_SIZE)) + def _prepare_migration_flags(self): migration_flags = 0 @@ -6176,6 +6194,14 @@ class LibvirtDriver(driver.ComputeDriver): if not CONF.workarounds.libvirt_disable_apic: guest.add_feature(vconfig.LibvirtConfigGuestFeatureAPIC()) + if ( + CONF.libvirt.virt_type == 'qemu' and + CONF.libvirt.tb_cache_size and + CONF.libvirt.tb_cache_size > 0 + ): + guest.add_feature(vconfig.LibvirtConfigGuestFeatureTCG( + CONF.libvirt.tb_cache_size)) + if CONF.libvirt.virt_type in ('qemu', 'kvm') and os_type == 'windows': hv = vconfig.LibvirtConfigGuestFeatureHyperV() hv.relaxed = True diff --git a/releasenotes/notes/libvirt-tb-cache-size-c0d0d672c0950393.yaml b/releasenotes/notes/libvirt-tb-cache-size-c0d0d672c0950393.yaml new file mode 100644 index 000000000000..b77653b37a11 --- /dev/null +++ b/releasenotes/notes/libvirt-tb-cache-size-c0d0d672c0950393.yaml @@ -0,0 +1,12 @@ +--- +features: + - | + Qemu>=5.0.0 bumped the default tb-cache size to 1GiB(from 32MiB) and this + made it difficult to run multiple guest VMs on systems running with lower + memory. With Libvirt>=8.0.0 it's possible to configure lower tb-cache + size. A new config option is introduced: + + ``[libvirt]tb_cache_size`` + + This config option can be used to configure tb_cache size + for guest VMs, it's only applicable with ``virt_type=qemu``.