From 76d64b9cb4241b73e62b3775f13d8eddcc0cb778 Mon Sep 17 00:00:00 2001 From: elajkat Date: Tue, 17 Dec 2024 11:00:44 +0100 Subject: [PATCH] blueprint: iothreads-for-instances Enable one io-thread per qemu instance. Related-Bug: iothreads-for-instances Change-Id: I8b22e5bca560d111934fbdf67494a4e288b9e50a Signed-off-by: lajoskatona --- nova/tests/unit/virt/libvirt/test_config.py | 55 +++++ nova/tests/unit/virt/libvirt/test_driver.py | 206 ++++++++++++++++++ nova/virt/libvirt/config.py | 29 +++ nova/virt/libvirt/driver.py | 11 + .../iothread-for-vms-1ba1b8c4189dd5a3.yaml | 4 + 5 files changed, 305 insertions(+) create mode 100644 releasenotes/notes/iothread-for-vms-1ba1b8c4189dd5a3.yaml diff --git a/nova/tests/unit/virt/libvirt/test_config.py b/nova/tests/unit/virt/libvirt/test_config.py index 3bd460bd4208..8bc1c8411684 100644 --- a/nova/tests/unit/virt/libvirt/test_config.py +++ b/nova/tests/unit/virt/libvirt/test_config.py @@ -2996,6 +2996,18 @@ class LibvirtConfigGuestTest(LibvirtConfigBaseTest): """) + def test_config_guest_iothreads(self): + xml = """ + + test42 + 1 + + """ + obj = config.LibvirtConfigGuest() + obj.parse_str(xml) + + self.assertEqual(1, obj.iothreads) + def test_ConfigGuest_parse_devices(self): xmldoc = """ @@ -3995,6 +4007,49 @@ class LibvirtConfigGuestCPUTuneTest(LibvirtConfigBaseTest): """) + def test_config_cputune_vcpus_iothread(self): + cputune = config.LibvirtConfigGuestCPUTune() + + vcpu0 = config.LibvirtConfigGuestCPUTuneVCPUPin() + vcpu0.id = 0 + vcpu0.cpuset = set([0, 1]) + vcpu1 = config.LibvirtConfigGuestCPUTuneVCPUPin() + vcpu1.id = 1 + vcpu1.cpuset = set([2, 3]) + vcpu2 = config.LibvirtConfigGuestCPUTuneVCPUPin() + vcpu2.id = 2 + vcpu2.cpuset = set([4, 5]) + vcpu3 = config.LibvirtConfigGuestCPUTuneVCPUPin() + vcpu3.id = 3 + vcpu3.cpuset = set([6, 7]) + cputune.vcpupin.extend([vcpu0, vcpu1, vcpu2, vcpu3]) + + emu = config.LibvirtConfigGuestCPUTuneIOThreadPin() + emu.cpuset = set([0, 1, 2, 3, 4, 5, 6, 7]) + cputune.emulatorpin = emu + + sch0 = config.LibvirtConfigGuestCPUTuneVCPUSched() + sch0.vcpus = set([0, 1, 2, 3]) + sch0.scheduler = "fifo" + sch0.priority = 1 + sch1 = config.LibvirtConfigGuestCPUTuneVCPUSched() + sch1.vcpus = set([4, 5, 6, 7]) + sch1.scheduler = "fifo" + sch1.priority = 99 + cputune.vcpusched.extend([sch0, sch1]) + + xml = cputune.to_xml() + self.assertXmlEqual(""" + + + + + + + + + """, xml) + class LibvirtConfigGuestMemoryBackingTest(LibvirtConfigBaseTest): def test_config_memory_backing_none(self): diff --git a/nova/tests/unit/virt/libvirt/test_driver.py b/nova/tests/unit/virt/libvirt/test_driver.py index 0e12e1c3e5ce..7ba862e4d5b0 100644 --- a/nova/tests/unit/virt/libvirt/test_driver.py +++ b/nova/tests/unit/virt/libvirt/test_driver.py @@ -5174,6 +5174,212 @@ class LibvirtConnTestCase(test.NoDBTestCase, self.assertEqual(set([7]), cfg.cputune.vcpupin[2].cpuset) self.assertEqual(set([8]), cfg.cputune.vcpupin[3].cpuset) + @mock.patch.object(host.Host, "_check_machine_type", new=mock.Mock()) + def test_get_guest_config_iothreads_instance(self): + self.flags(cpu_shared_set='0-5', cpu_dedicated_set=None, + group='compute') + + instance_topology = objects.InstanceNUMATopology(cells=[ + objects.InstanceNUMACell( + id=0, cpuset=set([0, 1, 2, 3]), pcpuset=set(), + memory=1024, pagesize=None), + ]) + instance_ref = objects.Instance(**self.test_instance) + instance_ref.numa_topology = instance_topology + image_meta = objects.ImageMeta.from_dict(self.test_image_meta) + flavor = objects.Flavor(memory_mb=2048, vcpus=4, root_gb=496, + ephemeral_gb=8128, swap=33550336, name='fake', + extra_specs={}, id=42, flavorid='someflavor') + instance_ref.flavor = flavor + + caps = vconfig.LibvirtConfigCaps() + caps.host = vconfig.LibvirtConfigCapsHost() + caps.host.cpu = vconfig.LibvirtConfigCPU() + caps.host.cpu.arch = fields.Architecture.X86_64 + caps.host.topology = fakelibvirt.NUMATopology() + + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) + disk_info = blockinfo.get_disk_info(CONF.libvirt.virt_type, + instance_ref, image_meta) + + with test.nested( + mock.patch.object( + objects.InstanceNUMATopology, "get_by_instance_uuid", + return_value=instance_topology), + mock.patch.object(host.Host, 'has_min_version', + return_value=True), + mock.patch.object(host.Host, "get_capabilities", + return_value=caps), + mock.patch.object(host.Host, 'get_online_cpus', + return_value=set(range(8))), + ): + cfg = drvr._get_guest_config(instance_ref, [], + image_meta, disk_info) + + self.assertEqual(1, cfg.iothreads) + + @mock.patch.object(host.Host, "_check_machine_type", new=mock.Mock()) + def test_get_guest_config_iothreadpin_matches_emulatorpin(self): + self.flags(cpu_shared_set=None, cpu_dedicated_set='4-8', + group='compute') + + instance_topology = objects.InstanceNUMATopology( + emulator_threads_policy=( + fields.CPUEmulatorThreadsPolicy.ISOLATE), + cells=[ + objects.InstanceNUMACell( + id=0, cpuset=set(), pcpuset=set([0, 1]), + memory=1024, pagesize=2048, + cpu_policy=fields.CPUAllocationPolicy.DEDICATED, + cpu_pinning={0: 4, 1: 5}, + cpuset_reserved=set([6])), + objects.InstanceNUMACell( + id=1, cpuset=set(), pcpuset=set([2, 3]), + memory=1024, pagesize=2048, + cpu_policy=fields.CPUAllocationPolicy.DEDICATED, + cpu_pinning={2: 7, 3: 8}), + ]) + + instance_ref = objects.Instance(**self.test_instance) + instance_ref.numa_topology = instance_topology + image_meta = objects.ImageMeta.from_dict(self.test_image_meta) + + caps = vconfig.LibvirtConfigCaps() + caps.host = vconfig.LibvirtConfigCapsHost() + caps.host.cpu = vconfig.LibvirtConfigCPU() + caps.host.cpu.arch = "x86_64" + caps.host.topology = fakelibvirt.NUMATopology() + + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) + disk_info = blockinfo.get_disk_info(CONF.libvirt.virt_type, + instance_ref, image_meta) + + with test.nested( + mock.patch.object( + objects.InstanceNUMATopology, "get_by_instance_uuid", + return_value=instance_topology), + mock.patch.object(host.Host, 'has_min_version', + return_value=True), + mock.patch.object(host.Host, "get_capabilities", + return_value=caps), + mock.patch.object(host.Host, 'get_online_cpus', + return_value=set(range(10))), + ): + cfg = drvr._get_guest_config(instance_ref, [], + image_meta, disk_info) + + self.assertIsInstance(cfg.cputune.iothreadpin, + vconfig.LibvirtConfigGuestCPUTuneIOThreadPin) + self.assertEqual(cfg.cputune.emulatorpin.cpuset, + cfg.cputune.iothreadpin.cpuset) + self.assertEqual(set([6]), cfg.cputune.iothreadpin.cpuset) + + @mock.patch.object(host.Host, "_check_machine_type", new=mock.Mock()) + def test_get_guest_config_iothreadpin_shared_emulator_threads(self): + self.flags(cpu_shared_set='1-2', cpu_dedicated_set='4-8', + group='compute') + + instance_topology = objects.InstanceNUMATopology( + emulator_threads_policy=( + fields.CPUEmulatorThreadsPolicy.SHARE), + cells=[ + objects.InstanceNUMACell( + id=0, cpuset=set(), pcpuset=set([0, 1]), + memory=1024, pagesize=2048, + cpu_policy=fields.CPUAllocationPolicy.DEDICATED, + cpu_pinning={0: 4, 1: 5}), + objects.InstanceNUMACell( + id=1, cpuset=set(), pcpuset=set([2, 3]), + memory=1024, pagesize=2048, + cpu_policy=fields.CPUAllocationPolicy.DEDICATED, + cpu_pinning={2: 7, 3: 8}), + ]) + + instance_ref = objects.Instance(**self.test_instance) + instance_ref.numa_topology = instance_topology + image_meta = objects.ImageMeta.from_dict(self.test_image_meta) + + caps = vconfig.LibvirtConfigCaps() + caps.host = vconfig.LibvirtConfigCapsHost() + caps.host.cpu = vconfig.LibvirtConfigCPU() + caps.host.cpu.arch = "x86_64" + caps.host.topology = fakelibvirt.NUMATopology() + + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) + disk_info = blockinfo.get_disk_info(CONF.libvirt.virt_type, + instance_ref, image_meta) + + with test.nested( + mock.patch.object( + objects.InstanceNUMATopology, "get_by_instance_uuid", + return_value=instance_topology), + mock.patch.object(host.Host, 'has_min_version', + return_value=True), + mock.patch.object(host.Host, "get_capabilities", + return_value=caps), + mock.patch.object(host.Host, 'get_online_cpus', + return_value=set(range(10))), + ): + cfg = drvr._get_guest_config(instance_ref, [], + image_meta, disk_info) + + self.assertIsInstance(cfg.cputune.iothreadpin, + vconfig.LibvirtConfigGuestCPUTuneIOThreadPin) + self.assertEqual(cfg.cputune.emulatorpin.cpuset, + cfg.cputune.iothreadpin.cpuset) + self.assertEqual(set([1, 2]), cfg.cputune.iothreadpin.cpuset) + + @mock.patch.object(host.Host, "_check_machine_type", new=mock.Mock()) + def test_get_guest_config_iothreadpin_numa_topology(self): + self.flags(cpu_shared_set='0-5', cpu_dedicated_set=None, + group='compute') + + instance_topology = objects.InstanceNUMATopology(cells=[ + objects.InstanceNUMACell( + id=0, cpuset=set([0, 1]), pcpuset=set(), memory=1024, + pagesize=None), + objects.InstanceNUMACell( + id=1, cpuset=set([2, 3]), pcpuset=set(), memory=1024, + pagesize=None), + ]) + instance_ref = objects.Instance(**self.test_instance) + instance_ref.numa_topology = instance_topology + image_meta = objects.ImageMeta.from_dict(self.test_image_meta) + flavor = objects.Flavor(memory_mb=2048, vcpus=4, root_gb=496, + ephemeral_gb=8128, swap=33550336, name='fake', + extra_specs={}, id=42, flavorid='someflavor') + instance_ref.flavor = flavor + + caps = vconfig.LibvirtConfigCaps() + caps.host = vconfig.LibvirtConfigCapsHost() + caps.host.cpu = vconfig.LibvirtConfigCPU() + caps.host.cpu.arch = fields.Architecture.X86_64 + caps.host.topology = fakelibvirt.NUMATopology() + + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) + disk_info = blockinfo.get_disk_info(CONF.libvirt.virt_type, + instance_ref, image_meta) + + with test.nested( + mock.patch.object( + objects.InstanceNUMATopology, "get_by_instance_uuid", + return_value=instance_topology), + mock.patch.object(host.Host, 'has_min_version', + return_value=True), + mock.patch.object(host.Host, "get_capabilities", + return_value=caps), + mock.patch.object(host.Host, 'get_online_cpus', + return_value=set(range(8))), + ): + cfg = drvr._get_guest_config(instance_ref, [], + image_meta, disk_info) + + self.assertIsInstance(cfg.cputune.iothreadpin, + vconfig.LibvirtConfigGuestCPUTuneIOThreadPin) + self.assertEqual(cfg.cputune.emulatorpin.cpuset, + cfg.cputune.iothreadpin.cpuset) + self.assertEqual(set([0, 1, 2, 3]), cfg.cputune.iothreadpin.cpuset) + def test_get_guest_config_numa_host_instance_shared_emulthreads_err( self): self.flags(cpu_shared_set='48-50', cpu_dedicated_set='4-8', diff --git a/nova/virt/libvirt/config.py b/nova/virt/libvirt/config.py index 33a5bad4ef5b..a76ead794b49 100644 --- a/nova/virt/libvirt/config.py +++ b/nova/virt/libvirt/config.py @@ -2664,6 +2664,25 @@ class LibvirtConfigGuestCPUTuneEmulatorPin(LibvirtConfigObject): return root +class LibvirtConfigGuestCPUTuneIOThreadPin(LibvirtConfigObject): + + def __init__(self, **kwargs): + super(LibvirtConfigGuestCPUTuneIOThreadPin, self).__init__( + root_name="iothreadpin", + **kwargs) + + self.cpuset = None + + def format_dom(self): + root = super(LibvirtConfigGuestCPUTuneIOThreadPin, self).format_dom() + + if self.cpuset is not None: + root.set("cpuset", + hardware.format_cpu_spec(self.cpuset)) + + return root + + class LibvirtConfigGuestCPUTuneVCPUSched(LibvirtConfigObject): def __init__(self, **kwargs): @@ -2699,6 +2718,7 @@ class LibvirtConfigGuestCPUTune(LibvirtConfigObject): self.period = None self.vcpupin = [] self.emulatorpin = None + self.iothreadpin = None self.vcpusched = [] def format_dom(self): @@ -2713,6 +2733,8 @@ class LibvirtConfigGuestCPUTune(LibvirtConfigObject): if self.emulatorpin is not None: root.append(self.emulatorpin.format_dom()) + if self.iothreadpin is not None: + root.append(self.iothreadpin.format_dom()) for vcpu in self.vcpupin: root.append(vcpu.format_dom()) for sched in self.vcpusched: @@ -3092,6 +3114,7 @@ class LibvirtConfigGuest(LibvirtConfigObject): self.cpuset = None self.cpu = None self.cputune = None + self.iothreads = None self.features = [] self.clock = None self.sysinfo = None @@ -3261,6 +3284,10 @@ class LibvirtConfigGuest(LibvirtConfigObject): self._format_os(root) self._format_features(root) + # Set 1 IO thread per VM for all instances. + if self.iothreads is not None: + root.append(self._text_node("iothreads", str(self.iothreads))) + if self.cputune is not None: root.append(self.cputune.format_dom()) @@ -3390,6 +3417,8 @@ class LibvirtConfigGuest(LibvirtConfigObject): self.add_perf_event(p.get('name')) elif c.tag == 'os': self._parse_os(c) + elif c.tag == 'iothreads': + self.iothreads = int(c.text) else: self._parse_basic_props(c) diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index d6f089bdf403..ff707f2214b4 100644 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -6490,6 +6490,9 @@ class LibvirtDriver(driver.ComputeDriver): guest_cpu_tune.emulatorpin = ( vconfig.LibvirtConfigGuestCPUTuneEmulatorPin()) guest_cpu_tune.emulatorpin.cpuset = set([]) + guest_cpu_tune.iothreadpin = ( + vconfig.LibvirtConfigGuestCPUTuneIOThreadPin()) + guest_cpu_tune.iothreadpin.cpuset = set([]) # Init NUMATune configuration guest_numa_tune = vconfig.LibvirtConfigGuestNUMATune() @@ -6528,7 +6531,12 @@ class LibvirtDriver(driver.ComputeDriver): emu_pin_cpuset = self._get_emulatorpin_cpuset( cpu, object_numa_cell, vcpus_rt, emulator_threads_policy, pin_cpuset) + # Note(lajoskatona): Here we set emu_pin_cpuset for both + # emulatorpin and iothreadpin, this is makes sure that + # both emulator and iothreads are pinned to cores other + # than the instance's cores to support realtime cpus. guest_cpu_tune.emulatorpin.cpuset.update(emu_pin_cpuset) + guest_cpu_tune.iothreadpin.cpuset.update(emu_pin_cpuset) # TODO(berrange) When the guest has >1 NUMA node, it will # span multiple host NUMA nodes. By pinning emulator threads @@ -7559,6 +7567,9 @@ class LibvirtDriver(driver.ComputeDriver): self._set_features(guest, instance.os_type, image_meta, flavor) self._set_clock(guest, instance.os_type, image_meta) + # Set IOThreads to 1 for everybody + guest.iothreads = 1 + storage_configs = self._get_guest_storage_config(context, instance, image_meta, disk_info, rescue, block_device_info, flavor, guest.os_type) diff --git a/releasenotes/notes/iothread-for-vms-1ba1b8c4189dd5a3.yaml b/releasenotes/notes/iothread-for-vms-1ba1b8c4189dd5a3.yaml new file mode 100644 index 000000000000..729f8b65fb5e --- /dev/null +++ b/releasenotes/notes/iothread-for-vms-1ba1b8c4189dd5a3.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Enable IOThread for Virtual Machines, for now 1 per VM.