From b6aef1ec4f9848f85e1f367e560c2bdb703fa110 Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Tue, 28 Jul 2020 16:22:39 +0100 Subject: [PATCH] Handle multiple 'vcpusched' elements during live migrate When live migrating a pinned instance, we recalculate pinning information for the destination host and then update the instance's XML before spawning the instance there. As part of the pinning information recalculation, we must also recalculate information for realtime cores, which are configured using the '' element. The 'nova.virt.libvirt.migration._update_numa_xml' function, which handles this updating, was assuming there would only be one of these elements. This is a reasonably sane assumption since this is all we create in the 'nova.virt.libvirt.LibvirtDriver._get_guest_numa_config' function used to generate the initial instance XML. However, a look at logs show that at least some (all?) versions of libvirt actually rewrite the XML we're providing them. Compare what is returned from '_get_guest_xml': DEBUG nova.virt.libvirt.driver [...] [instance: ...] End _get_guest_xml xml= ... 4096 ... ... {{(pid=12600) _get_guest_xml /opt/stack/nova/nova/virt/libvirt/driver.py:6331}} to what is seen when we enter '_update_numa_xml' (or via 'virsh dumpxml' at any point after instance creation): DEBUG nova.virt.libvirt.migration [-] _update_numa_xml input xml= ... 4096 ... {{(pid=12600) _update_numa_xml /opt/stack/nova/nova/virt/libvirt/migration.py:97} The solution is simple: rather than trying to modify the existing XML, simply scrap it and rebuild the elements from scratch. We should probably do this for all elements, but that can/should be tackled separately. Change-Id: Ic01603a91f6099f1068af0e955f3e1056021d673 Signed-off-by: Stephen Finucane Closes-Bug: #1889257 --- .../tests/unit/virt/libvirt/test_migration.py | 65 ++++++++++--------- nova/virt/libvirt/migration.py | 20 +++++- 2 files changed, 53 insertions(+), 32 deletions(-) diff --git a/nova/tests/unit/virt/libvirt/test_migration.py b/nova/tests/unit/virt/libvirt/test_migration.py index 7f21a5bd096c..d4fba48397de 100644 --- a/nova/tests/unit/virt/libvirt/test_migration.py +++ b/nova/tests/unit/virt/libvirt/test_migration.py @@ -153,42 +153,49 @@ class UtilityMigrationTestCase(test.NoDBTestCase): self.assertXmlEqual(res, new_xml) def test_update_numa_xml(self): - xml = textwrap.dedent(""" + doc = etree.fromstring(""" - - - - - - - - - - - + + + + + + + + + + + + """) - doc = etree.fromstring(xml) data = objects.LibvirtLiveMigrateData( dst_numa_info=objects.LibvirtLiveMigrateNUMAInfo( - cpu_pins={'0': set([10, 11]), - '1': set([12, 13])}, - cell_pins={'2': set([14, 15]), - '3': set([16, 17])}, + cpu_pins={'0': set([10, 11]), '1': set([12, 13])}, + cell_pins={'2': set([14, 15]), '3': set([16, 17])}, emulator_pins=set([18, 19]), sched_vcpus=set([20, 21]), sched_priority=22)) - res = etree.tostring(migration._update_numa_xml(copy.deepcopy(doc), - data), - encoding='unicode') - doc.find('./cputune/vcpupin/[@vcpu="0"]').set('cpuset', '10-11') - doc.find('./cputune/vcpupin/[@vcpu="1"]').set('cpuset', '12-13') - doc.find('./cputune/emulatorpin').set('cpuset', '18-19') - doc.find('./cputune/vcpusched').set('vcpus', '20-21') - doc.find('./cputune/vcpusched').set('priority', '22') - doc.find('./numatune/memory').set('nodeset', '14-17') - doc.find('./numatune/memnode/[@cellid="2"]').set('nodeset', '14-15') - doc.find('./numatune/memnode/[@cellid="3"]').set('nodeset', '16-17') - self.assertXmlEqual(res, etree.tostring(doc, encoding='unicode')) + + result = etree.tostring( + migration._update_numa_xml(copy.deepcopy(doc), data), + encoding='unicode') + + expected = textwrap.dedent(""" + + + + + + + + + + + + + """) + + self.assertXmlEqual(expected, result) def test_update_numa_xml_no_updates(self): xml = textwrap.dedent(""" diff --git a/nova/virt/libvirt/migration.py b/nova/virt/libvirt/migration.py index 8b79f1979528..9543adda7ad5 100644 --- a/nova/virt/libvirt/migration.py +++ b/nova/virt/libvirt/migration.py @@ -123,9 +123,23 @@ def _update_numa_xml(xml_doc, migrate_data): memory.set('nodeset', hardware.format_cpu_spec(set(all_cells))) if 'sched_vcpus' and 'sched_priority' in info: - vcpusched = xml_doc.find('./cputune/vcpusched') - vcpusched.set('vcpus', hardware.format_cpu_spec(info.sched_vcpus)) - vcpusched.set('priority', str(info.sched_priority)) + cputune = xml_doc.find('./cputune') + + # delete the old variant(s) + for elem in cputune.findall('./vcpusched'): + elem.getparent().remove(elem) + + # ...and create a new, shiny one + vcpusched = vconfig.LibvirtConfigGuestCPUTuneVCPUSched() + vcpusched.vcpus = info.sched_vcpus + vcpusched.priority = info.sched_priority + # TODO(stephenfin): Stop assuming scheduler type. We currently only + # create these elements for real-time instances and 'fifo' is the only + # scheduler policy we currently support so this is reasonably safe to + # assume, but it's still unnecessary + vcpusched.scheduler = 'fifo' + + cputune.append(vcpusched.format_dom()) LOG.debug('_update_numa_xml output xml=%s', etree.tostring(xml_doc, encoding='unicode', pretty_print=True))