From ac05bc3b38684e47e35e06a1514866773d8f3609 Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Tue, 28 Jul 2020 16:22:39 +0100 Subject: [PATCH] Handle multiple 'vcpusched' elements during live migrate When live migrating a pinned instance, we recalculate pinning information for the destination host and then update the instance's XML before spawning the instance there. As part of the pinning information recalculation, we must also recalculate information for realtime cores, which are configured using the '' element. The 'nova.virt.libvirt.migration._update_numa_xml' function, which handles this updating, was assuming there would only be one of these elements. This is a reasonably sane assumption since this is all we create in the 'nova.virt.libvirt.LibvirtDriver._get_guest_numa_config' function used to generate the initial instance XML. However, a look at logs show that at least some (all?) versions of libvirt actually rewrite the XML we're providing them. Compare what is returned from '_get_guest_xml': DEBUG nova.virt.libvirt.driver [...] [instance: ...] End _get_guest_xml xml= ... 4096 ... ... {{(pid=12600) _get_guest_xml /opt/stack/nova/nova/virt/libvirt/driver.py:6331}} to what is seen when we enter '_update_numa_xml' (or via 'virsh dumpxml' at any point after instance creation): DEBUG nova.virt.libvirt.migration [-] _update_numa_xml input xml= ... 4096 ... {{(pid=12600) _update_numa_xml /opt/stack/nova/nova/virt/libvirt/migration.py:97} The solution is simple: rather than trying to modify the existing XML, simply scrap it and rebuild the elements from scratch. We should probably do this for all elements, but that can/should be tackled separately. Change-Id: Ic01603a91f6099f1068af0e955f3e1056021d673 Signed-off-by: Stephen Finucane Closes-Bug: #1889257 (cherry picked from commit b6aef1ec4f9848f85e1f367e560c2bdb703fa110) --- .../tests/unit/virt/libvirt/test_migration.py | 65 ++++++++++--------- nova/virt/libvirt/migration.py | 20 +++++- 2 files changed, 53 insertions(+), 32 deletions(-) diff --git a/nova/tests/unit/virt/libvirt/test_migration.py b/nova/tests/unit/virt/libvirt/test_migration.py index 39321b912502..b849170230ca 100644 --- a/nova/tests/unit/virt/libvirt/test_migration.py +++ b/nova/tests/unit/virt/libvirt/test_migration.py @@ -181,42 +181,49 @@ class UtilityMigrationTestCase(test.NoDBTestCase): self.assertXmlEqual(res, new_xml) def test_update_numa_xml(self): - xml = textwrap.dedent(""" + doc = etree.fromstring(""" - - - - - - - - - - - + + + + + + + + + + + + """) - doc = etree.fromstring(xml) data = objects.LibvirtLiveMigrateData( dst_numa_info=objects.LibvirtLiveMigrateNUMAInfo( - cpu_pins={'0': set([10, 11]), - '1': set([12, 13])}, - cell_pins={'2': set([14, 15]), - '3': set([16, 17])}, + cpu_pins={'0': set([10, 11]), '1': set([12, 13])}, + cell_pins={'2': set([14, 15]), '3': set([16, 17])}, emulator_pins=set([18, 19]), sched_vcpus=set([20, 21]), sched_priority=22)) - res = etree.tostring(migration._update_numa_xml(copy.deepcopy(doc), - data), - encoding='unicode') - doc.find('./cputune/vcpupin/[@vcpu="0"]').set('cpuset', '10-11') - doc.find('./cputune/vcpupin/[@vcpu="1"]').set('cpuset', '12-13') - doc.find('./cputune/emulatorpin').set('cpuset', '18-19') - doc.find('./cputune/vcpusched').set('vcpus', '20-21') - doc.find('./cputune/vcpusched').set('priority', '22') - doc.find('./numatune/memory').set('nodeset', '14-17') - doc.find('./numatune/memnode/[@cellid="2"]').set('nodeset', '14-15') - doc.find('./numatune/memnode/[@cellid="3"]').set('nodeset', '16-17') - self.assertXmlEqual(res, etree.tostring(doc, encoding='unicode')) + + result = etree.tostring( + migration._update_numa_xml(copy.deepcopy(doc), data), + encoding='unicode') + + expected = textwrap.dedent(""" + + + + + + + + + + + + + """) + + self.assertXmlEqual(expected, result) def test_update_numa_xml_no_updates(self): xml = textwrap.dedent(""" diff --git a/nova/virt/libvirt/migration.py b/nova/virt/libvirt/migration.py index 4143261ad038..a3bc92c527df 100644 --- a/nova/virt/libvirt/migration.py +++ b/nova/virt/libvirt/migration.py @@ -150,9 +150,23 @@ def _update_numa_xml(xml_doc, migrate_data): memory.set('nodeset', hardware.format_cpu_spec(set(all_cells))) if 'sched_vcpus' and 'sched_priority' in info: - vcpusched = xml_doc.find('./cputune/vcpusched') - vcpusched.set('vcpus', hardware.format_cpu_spec(info.sched_vcpus)) - vcpusched.set('priority', str(info.sched_priority)) + cputune = xml_doc.find('./cputune') + + # delete the old variant(s) + for elem in cputune.findall('./vcpusched'): + elem.getparent().remove(elem) + + # ...and create a new, shiny one + vcpusched = vconfig.LibvirtConfigGuestCPUTuneVCPUSched() + vcpusched.vcpus = info.sched_vcpus + vcpusched.priority = info.sched_priority + # TODO(stephenfin): Stop assuming scheduler type. We currently only + # create these elements for real-time instances and 'fifo' is the only + # scheduler policy we currently support so this is reasonably safe to + # assume, but it's still unnecessary + vcpusched.scheduler = 'fifo' + + cputune.append(vcpusched.format_dom()) LOG.debug('_update_numa_xml output xml=%s', etree.tostring(xml_doc, encoding='unicode', pretty_print=True))