openstack/nova
Code Issues Proposed changes

blueprint: iothreads-for-instances

Browse Source 
Enable one io-thread per qemu instance.

Related-Bug: iothreads-for-instances
Change-Id: I8b22e5bca560d111934fbdf67494a4e288b9e50a
Signed-off-by: lajoskatona <lajos.katona@est.tech>
This commit is contained in:
elajkat
2024-12-17 11:00:44 +01:00
committed by lajoskatona
parent 66bad0b1d9
commit 76d64b9cb4
5 changed files with 305 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
nova/tests/unit/virt/libvirt/test_config.py
View File

@@ -2996,6 +2996,18 @@ class LibvirtConfigGuestTest(LibvirtConfigBaseTest):
</os>
</domain>""")
def test_config_guest_iothreads(self):
xml = """
<domain type="kvm">
<name>test42</name>
<iothreads>1</iothreads>
</domain>
"""
obj = config.LibvirtConfigGuest()
obj.parse_str(xml)
self.assertEqual(1, obj.iothreads)
def test_ConfigGuest_parse_devices(self):
xmldoc = """ <domain type="kvm">
<devices>
@@ -3995,6 +4007,49 @@ class LibvirtConfigGuestCPUTuneTest(LibvirtConfigBaseTest):
<vcpusched vcpus="4-7" scheduler="fifo" priority="99"/>
</cputune>""")
def test_config_cputune_vcpus_iothread(self):
cputune = config.LibvirtConfigGuestCPUTune()
vcpu0 = config.LibvirtConfigGuestCPUTuneVCPUPin()
vcpu0.id = 0
vcpu0.cpuset = set([0, 1])
vcpu1 = config.LibvirtConfigGuestCPUTuneVCPUPin()
vcpu1.id = 1
vcpu1.cpuset = set([2, 3])
vcpu2 = config.LibvirtConfigGuestCPUTuneVCPUPin()
vcpu2.id = 2
vcpu2.cpuset = set([4, 5])
vcpu3 = config.LibvirtConfigGuestCPUTuneVCPUPin()
vcpu3.id = 3
vcpu3.cpuset = set([6, 7])
cputune.vcpupin.extend([vcpu0, vcpu1, vcpu2, vcpu3])
emu = config.LibvirtConfigGuestCPUTuneIOThreadPin()
emu.cpuset = set([0, 1, 2, 3, 4, 5, 6, 7])
cputune.emulatorpin = emu
sch0 = config.LibvirtConfigGuestCPUTuneVCPUSched()
sch0.vcpus = set([0, 1, 2, 3])
sch0.scheduler = "fifo"
sch0.priority = 1
sch1 = config.LibvirtConfigGuestCPUTuneVCPUSched()
sch1.vcpus = set([4, 5, 6, 7])
sch1.scheduler = "fifo"
sch1.priority = 99
cputune.vcpusched.extend([sch0, sch1])
xml = cputune.to_xml()
self.assertXmlEqual("""
<cputune>
<iothreadpin cpuset="0-7"/>
<vcpupin vcpu="0" cpuset="0-1"/>
<vcpupin vcpu="1" cpuset="2-3"/>
<vcpupin vcpu="2" cpuset="4-5"/>
<vcpupin vcpu="3" cpuset="6-7"/>
<vcpusched vcpus="0-3" scheduler="fifo" priority="1"/>
<vcpusched vcpus="4-7" scheduler="fifo" priority="99"/>
</cputune>""", xml)
class LibvirtConfigGuestMemoryBackingTest(LibvirtConfigBaseTest):
def test_config_memory_backing_none(self):

206
nova/tests/unit/virt/libvirt/test_driver.py
View File

@@ -5174,6 +5174,212 @@ class LibvirtConnTestCase(test.NoDBTestCase,
self.assertEqual(set([7]), cfg.cputune.vcpupin[2].cpuset)
self.assertEqual(set([8]), cfg.cputune.vcpupin[3].cpuset)
@mock.patch.object(host.Host, "_check_machine_type", new=mock.Mock())
def test_get_guest_config_iothreads_instance(self):
self.flags(cpu_shared_set='0-5', cpu_dedicated_set=None,
group='compute')
instance_topology = objects.InstanceNUMATopology(cells=[
objects.InstanceNUMACell(
id=0, cpuset=set([0, 1, 2, 3]), pcpuset=set(),
memory=1024, pagesize=None),
])
instance_ref = objects.Instance(**self.test_instance)
instance_ref.numa_topology = instance_topology
image_meta = objects.ImageMeta.from_dict(self.test_image_meta)
flavor = objects.Flavor(memory_mb=2048, vcpus=4, root_gb=496,
ephemeral_gb=8128, swap=33550336, name='fake',
extra_specs={}, id=42, flavorid='someflavor')
instance_ref.flavor = flavor
caps = vconfig.LibvirtConfigCaps()
caps.host = vconfig.LibvirtConfigCapsHost()
caps.host.cpu = vconfig.LibvirtConfigCPU()
caps.host.cpu.arch = fields.Architecture.X86_64
caps.host.topology = fakelibvirt.NUMATopology()
drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True)
disk_info = blockinfo.get_disk_info(CONF.libvirt.virt_type,
instance_ref, image_meta)
with test.nested(
mock.patch.object(
objects.InstanceNUMATopology, "get_by_instance_uuid",
return_value=instance_topology),
mock.patch.object(host.Host, 'has_min_version',
return_value=True),
mock.patch.object(host.Host, "get_capabilities",
return_value=caps),
mock.patch.object(host.Host, 'get_online_cpus',
return_value=set(range(8))),
):
cfg = drvr._get_guest_config(instance_ref, [],
image_meta, disk_info)
self.assertEqual(1, cfg.iothreads)
@mock.patch.object(host.Host, "_check_machine_type", new=mock.Mock())
def test_get_guest_config_iothreadpin_matches_emulatorpin(self):
self.flags(cpu_shared_set=None, cpu_dedicated_set='4-8',
group='compute')
instance_topology = objects.InstanceNUMATopology(
emulator_threads_policy=(
fields.CPUEmulatorThreadsPolicy.ISOLATE),
cells=[
objects.InstanceNUMACell(
id=0, cpuset=set(), pcpuset=set([0, 1]),
memory=1024, pagesize=2048,
cpu_policy=fields.CPUAllocationPolicy.DEDICATED,
cpu_pinning={0: 4, 1: 5},
cpuset_reserved=set([6])),
objects.InstanceNUMACell(
id=1, cpuset=set(), pcpuset=set([2, 3]),
memory=1024, pagesize=2048,
cpu_policy=fields.CPUAllocationPolicy.DEDICATED,
cpu_pinning={2: 7, 3: 8}),
])
instance_ref = objects.Instance(**self.test_instance)
instance_ref.numa_topology = instance_topology
image_meta = objects.ImageMeta.from_dict(self.test_image_meta)
caps = vconfig.LibvirtConfigCaps()
caps.host = vconfig.LibvirtConfigCapsHost()
caps.host.cpu = vconfig.LibvirtConfigCPU()
caps.host.cpu.arch = "x86_64"
caps.host.topology = fakelibvirt.NUMATopology()
drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True)
disk_info = blockinfo.get_disk_info(CONF.libvirt.virt_type,
instance_ref, image_meta)
with test.nested(
mock.patch.object(
objects.InstanceNUMATopology, "get_by_instance_uuid",
return_value=instance_topology),
mock.patch.object(host.Host, 'has_min_version',
return_value=True),
mock.patch.object(host.Host, "get_capabilities",
return_value=caps),
mock.patch.object(host.Host, 'get_online_cpus',
return_value=set(range(10))),
):
cfg = drvr._get_guest_config(instance_ref, [],
image_meta, disk_info)
self.assertIsInstance(cfg.cputune.iothreadpin,
vconfig.LibvirtConfigGuestCPUTuneIOThreadPin)
self.assertEqual(cfg.cputune.emulatorpin.cpuset,
cfg.cputune.iothreadpin.cpuset)
self.assertEqual(set([6]), cfg.cputune.iothreadpin.cpuset)
@mock.patch.object(host.Host, "_check_machine_type", new=mock.Mock())
def test_get_guest_config_iothreadpin_shared_emulator_threads(self):
self.flags(cpu_shared_set='1-2', cpu_dedicated_set='4-8',
group='compute')
instance_topology = objects.InstanceNUMATopology(
emulator_threads_policy=(
fields.CPUEmulatorThreadsPolicy.SHARE),
cells=[
objects.InstanceNUMACell(
id=0, cpuset=set(), pcpuset=set([0, 1]),
memory=1024, pagesize=2048,
cpu_policy=fields.CPUAllocationPolicy.DEDICATED,
cpu_pinning={0: 4, 1: 5}),
objects.InstanceNUMACell(
id=1, cpuset=set(), pcpuset=set([2, 3]),
memory=1024, pagesize=2048,
cpu_policy=fields.CPUAllocationPolicy.DEDICATED,
cpu_pinning={2: 7, 3: 8}),
])
instance_ref = objects.Instance(**self.test_instance)
instance_ref.numa_topology = instance_topology
image_meta = objects.ImageMeta.from_dict(self.test_image_meta)
caps = vconfig.LibvirtConfigCaps()
caps.host = vconfig.LibvirtConfigCapsHost()
caps.host.cpu = vconfig.LibvirtConfigCPU()
caps.host.cpu.arch = "x86_64"
caps.host.topology = fakelibvirt.NUMATopology()
drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True)
disk_info = blockinfo.get_disk_info(CONF.libvirt.virt_type,
instance_ref, image_meta)
with test.nested(
mock.patch.object(
objects.InstanceNUMATopology, "get_by_instance_uuid",
return_value=instance_topology),
mock.patch.object(host.Host, 'has_min_version',
return_value=True),
mock.patch.object(host.Host, "get_capabilities",
return_value=caps),
mock.patch.object(host.Host, 'get_online_cpus',
return_value=set(range(10))),
):
cfg = drvr._get_guest_config(instance_ref, [],
image_meta, disk_info)
self.assertIsInstance(cfg.cputune.iothreadpin,
vconfig.LibvirtConfigGuestCPUTuneIOThreadPin)
self.assertEqual(cfg.cputune.emulatorpin.cpuset,
cfg.cputune.iothreadpin.cpuset)
self.assertEqual(set([1, 2]), cfg.cputune.iothreadpin.cpuset)
@mock.patch.object(host.Host, "_check_machine_type", new=mock.Mock())
def test_get_guest_config_iothreadpin_numa_topology(self):
self.flags(cpu_shared_set='0-5', cpu_dedicated_set=None,
group='compute')
instance_topology = objects.InstanceNUMATopology(cells=[
objects.InstanceNUMACell(
id=0, cpuset=set([0, 1]), pcpuset=set(), memory=1024,
pagesize=None),
objects.InstanceNUMACell(
id=1, cpuset=set([2, 3]), pcpuset=set(), memory=1024,
pagesize=None),
])
instance_ref = objects.Instance(**self.test_instance)
instance_ref.numa_topology = instance_topology
image_meta = objects.ImageMeta.from_dict(self.test_image_meta)
flavor = objects.Flavor(memory_mb=2048, vcpus=4, root_gb=496,
ephemeral_gb=8128, swap=33550336, name='fake',
extra_specs={}, id=42, flavorid='someflavor')
instance_ref.flavor = flavor
caps = vconfig.LibvirtConfigCaps()
caps.host = vconfig.LibvirtConfigCapsHost()
caps.host.cpu = vconfig.LibvirtConfigCPU()
caps.host.cpu.arch = fields.Architecture.X86_64
caps.host.topology = fakelibvirt.NUMATopology()
drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True)
disk_info = blockinfo.get_disk_info(CONF.libvirt.virt_type,
instance_ref, image_meta)
with test.nested(
mock.patch.object(
objects.InstanceNUMATopology, "get_by_instance_uuid",
return_value=instance_topology),
mock.patch.object(host.Host, 'has_min_version',
return_value=True),
mock.patch.object(host.Host, "get_capabilities",
return_value=caps),
mock.patch.object(host.Host, 'get_online_cpus',
return_value=set(range(8))),
):
cfg = drvr._get_guest_config(instance_ref, [],
image_meta, disk_info)
self.assertIsInstance(cfg.cputune.iothreadpin,
vconfig.LibvirtConfigGuestCPUTuneIOThreadPin)
self.assertEqual(cfg.cputune.emulatorpin.cpuset,
cfg.cputune.iothreadpin.cpuset)
self.assertEqual(set([0, 1, 2, 3]), cfg.cputune.iothreadpin.cpuset)
def test_get_guest_config_numa_host_instance_shared_emulthreads_err(
self):
self.flags(cpu_shared_set='48-50', cpu_dedicated_set='4-8',

29
nova/virt/libvirt/config.py
View File

@@ -2664,6 +2664,25 @@ class LibvirtConfigGuestCPUTuneEmulatorPin(LibvirtConfigObject):
return root
class LibvirtConfigGuestCPUTuneIOThreadPin(LibvirtConfigObject):
def __init__(self, **kwargs):
super(LibvirtConfigGuestCPUTuneIOThreadPin, self).__init__(
root_name="iothreadpin",
**kwargs)
self.cpuset = None
def format_dom(self):
root = super(LibvirtConfigGuestCPUTuneIOThreadPin, self).format_dom()
if self.cpuset is not None:
root.set("cpuset",
hardware.format_cpu_spec(self.cpuset))
return root
class LibvirtConfigGuestCPUTuneVCPUSched(LibvirtConfigObject):
def __init__(self, **kwargs):
@@ -2699,6 +2718,7 @@ class LibvirtConfigGuestCPUTune(LibvirtConfigObject):
self.period = None
self.vcpupin = []
self.emulatorpin = None
self.iothreadpin = None
self.vcpusched = []
def format_dom(self):
@@ -2713,6 +2733,8 @@ class LibvirtConfigGuestCPUTune(LibvirtConfigObject):
if self.emulatorpin is not None:
root.append(self.emulatorpin.format_dom())
if self.iothreadpin is not None:
root.append(self.iothreadpin.format_dom())
for vcpu in self.vcpupin:
root.append(vcpu.format_dom())
for sched in self.vcpusched:
@@ -3092,6 +3114,7 @@ class LibvirtConfigGuest(LibvirtConfigObject):
self.cpuset = None
self.cpu = None
self.cputune = None
self.iothreads = None
self.features = []
self.clock = None
self.sysinfo = None
@@ -3261,6 +3284,10 @@ class LibvirtConfigGuest(LibvirtConfigObject):
self._format_os(root)
self._format_features(root)
# Set 1 IO thread per VM for all instances.
if self.iothreads is not None:
root.append(self._text_node("iothreads", str(self.iothreads)))
if self.cputune is not None:
root.append(self.cputune.format_dom())
@@ -3390,6 +3417,8 @@ class LibvirtConfigGuest(LibvirtConfigObject):
self.add_perf_event(p.get('name'))
elif c.tag == 'os':
self._parse_os(c)
elif c.tag == 'iothreads':
self.iothreads = int(c.text)
else:
self._parse_basic_props(c)

11
nova/virt/libvirt/driver.py
View File

@@ -6490,6 +6490,9 @@ class LibvirtDriver(driver.ComputeDriver):
guest_cpu_tune.emulatorpin = (
vconfig.LibvirtConfigGuestCPUTuneEmulatorPin())
guest_cpu_tune.emulatorpin.cpuset = set([])
guest_cpu_tune.iothreadpin = (
vconfig.LibvirtConfigGuestCPUTuneIOThreadPin())
guest_cpu_tune.iothreadpin.cpuset = set([])
# Init NUMATune configuration
guest_numa_tune = vconfig.LibvirtConfigGuestNUMATune()
@@ -6528,7 +6531,12 @@ class LibvirtDriver(driver.ComputeDriver):
emu_pin_cpuset = self._get_emulatorpin_cpuset(
cpu, object_numa_cell, vcpus_rt,
emulator_threads_policy, pin_cpuset)
# Note(lajoskatona): Here we set emu_pin_cpuset for both
# emulatorpin and iothreadpin, this is makes sure that
# both emulator and iothreads are pinned to cores other
# than the instance's cores to support realtime cpus.
guest_cpu_tune.emulatorpin.cpuset.update(emu_pin_cpuset)
guest_cpu_tune.iothreadpin.cpuset.update(emu_pin_cpuset)
# TODO(berrange) When the guest has >1 NUMA node, it will
# span multiple host NUMA nodes. By pinning emulator threads
@@ -7559,6 +7567,9 @@ class LibvirtDriver(driver.ComputeDriver):
self._set_features(guest, instance.os_type, image_meta, flavor)
self._set_clock(guest, instance.os_type, image_meta)
# Set IOThreads to 1 for everybody
guest.iothreads = 1
storage_configs = self._get_guest_storage_config(context,
instance, image_meta, disk_info, rescue, block_device_info,
flavor, guest.os_type)

4
releasenotes/notes/iothread-for-vms-1ba1b8c4189dd5a3.yaml Normal file
View File

@@ -0,0 +1,4 @@
---
features:
- |
Enable IOThread for Virtual Machines, for now 1 per VM.