From f5aa144593d724565847eab534c835a82c3e3a34 Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Fri, 19 Jun 2020 12:20:36 +0100 Subject: [PATCH] hardware: Allow 'hw:cpu_realtime_mask' to be omitted By offloading emulator threads to other host cores (via the 'hw:emulator_threads_policy' extra spec), it's possible to allocate all guest cores to realtime. Enable this. Part of blueprint use-pcpu-and-vcpu-in-one-instance Change-Id: I00805cf9cca9657955c7e1ef3a76e384adaa78f1 Signed-off-by: Stephen Finucane --- doc/source/user/flavors.rst | 12 ++++++-- nova/exception.py | 5 ++-- nova/tests/unit/virt/libvirt/test_driver.py | 28 +++++++++++++++++++ nova/tests/unit/virt/test_hardware.py | 12 ++++++++ nova/virt/hardware.py | 11 +++++--- nova/virt/libvirt/driver.py | 22 +++++++++++++-- .../notes/bug-1884231-16acf297d88b122e.yaml | 5 ++++ 7 files changed, 84 insertions(+), 11 deletions(-) diff --git a/doc/source/user/flavors.rst b/doc/source/user/flavors.rst index cbeda92961c9..5d964493c6a2 100644 --- a/doc/source/user/flavors.rst +++ b/doc/source/user/flavors.rst @@ -601,8 +601,9 @@ CPU real-time policy .. important:: While most of your instance vCPUs can run with a real-time policy, you must - mark at least one vCPU as non-real-time, to be used for both non-real-time - guest processes and emulator overhead (housekeeping) processes. + either mark at least one vCPU as non-real-time to be account for emulator + overhead (housekeeping) or explicitly configure an :ref:`emulator thread + policy `. .. important:: @@ -633,6 +634,13 @@ CPU real-time policy The ``hw:cpu_realtime_mask`` option is only valid if ``hw:cpu_realtime`` is set to ``yes``. + .. versionchanged:: 22.0.0 (Victoria) + + Previously, it was necessary to specify ``hw:cpu_realtime_mask`` when + ``hw:cpu_realtime`` was set to yes. Starting in Victoria, it is possible + to omit this when an emulator thread policy is configured using the + ``hw:emulator_threads_policy`` extra spec. + .. _extra-specs-emulator-threads-policy: Emulator threads policy diff --git a/nova/exception.py b/nova/exception.py index 7cb2cd564f98..e0e1d80f9989 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -1894,9 +1894,8 @@ class LibguestfsCannotReadKernel(Invalid): class RealtimeMaskNotFoundOrInvalid(Invalid): - msg_fmt = _("Realtime policy needs vCPU(s) mask configured with at least " - "1 RT vCPU and 1 ordinary vCPU. See hw:cpu_realtime_mask " - "or hw_cpu_realtime_mask") + msg_fmt = _("Use of realtime CPUs requires either one or more " + "non-realtime CPU(s) or offloaded emulator threads.") class OsInfoNotFound(NotFound): diff --git a/nova/tests/unit/virt/libvirt/test_driver.py b/nova/tests/unit/virt/libvirt/test_driver.py index aa3c3f0e726a..f9be86a57159 100644 --- a/nova/tests/unit/virt/libvirt/test_driver.py +++ b/nova/tests/unit/virt/libvirt/test_driver.py @@ -3027,6 +3027,34 @@ class LibvirtConnTestCase(test.NoDBTestCase, self.assertTrue(membacking.locked) self.assertFalse(membacking.sharedpages) + def test_get_guest_memory_backing_config_realtime_invalid_share(self): + """Test behavior when there is no pool of shared CPUS on which to place + the emulator threads, isolating them from the instance CPU processes. + """ + extra_specs = { + "hw:cpu_realtime": "yes", + "hw:cpu_policy": "dedicated", + "hw:emulator_threads_policy": "share", + } + flavor = objects.Flavor( + name='m1.small', + memory_mb=6, + vcpus=28, + root_gb=496, + ephemeral_gb=8128, + swap=33550336, + extra_specs=extra_specs) + image_meta = objects.ImageMeta.from_dict(self.test_image_meta) + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), True) + + # this should fail because there is nowhere to place the emulator + # threads + self.assertRaises( + exception.RealtimeMaskNotFoundOrInvalid, + drvr._get_guest_memory_backing_config, + None, None, flavor, image_meta, + ) + def _test_sev_enabled(self, expected=None, host_sev_enabled=False, enc_extra_spec=None, enc_image_prop=None, hw_machine_type=None, hw_firmware_type=None): diff --git a/nova/tests/unit/virt/test_hardware.py b/nova/tests/unit/virt/test_hardware.py index 45c8ae052ac8..8888157db7ee 100644 --- a/nova/tests/unit/virt/test_hardware.py +++ b/nova/tests/unit/virt/test_hardware.py @@ -3887,6 +3887,18 @@ class CPURealtimeTestCase(test.NoDBTestCase): exception.RealtimeMaskNotFoundOrInvalid, hw.get_realtime_cpu_constraint, flavor, image) + def test_all_cpus_w_emulator_policy(self): + # The mask has no exclusion but there's an emulator thread policy + flavor = objects.Flavor( + vcpus=3, memory_mb=2048, extra_specs={ + 'hw:cpu_realtime': 'true', + 'hw:emulator_threads_policy': 'isolate' + }, + ) + image = objects.ImageMeta.from_dict({"properties": {}}) + rt = hw.get_realtime_cpu_constraint(flavor, image) + self.assertEqual({0, 1, 2}, rt) + def test_invalid_mask_rt_cpus_out_of_range(self): # The mask is not just an exclusion mask, and the RT range specifies # an invalid vCPU number. diff --git a/nova/virt/hardware.py b/nova/virt/hardware.py index bffeafac40f7..b8af9553eb6e 100644 --- a/nova/virt/hardware.py +++ b/nova/virt/hardware.py @@ -1705,16 +1705,19 @@ def get_realtime_cpu_constraint( # Image masks are used ahead of flavor masks as they will have more # specific requirements mask = image_mask or flavor_mask - if not mask: - raise exception.RealtimeMaskNotFoundOrInvalid() vcpus_set = set(range(flavor.vcpus)) - vcpus_rt = parse_cpu_spec("0-%d,%s" % (flavor.vcpus - 1, mask)) + if mask: + vcpus_rt = parse_cpu_spec("0-%d,%s" % (flavor.vcpus - 1, mask)) + else: + vcpus_rt = set(range(flavor.vcpus)) if not vcpus_rt: raise exception.RealtimeMaskNotFoundOrInvalid() - if vcpus_set == vcpus_rt: + # TODO(stephenfin): Do this check in numa_get_constraints instead + emu_policy = get_emulator_thread_policy_constraint(flavor) + if vcpus_set == vcpus_rt and not emu_policy: raise exception.RealtimeMaskNotFoundOrInvalid() if not vcpus_rt.issubset(vcpus_set): diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index 6b1a48a8eb51..4c174fa063f1 100644 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -5432,6 +5432,26 @@ class LibvirtDriver(driver.ComputeDriver): def _get_guest_memory_backing_config( self, inst_topology, numatune, flavor, image_meta): + wantsrealtime = hardware.is_realtime_enabled(flavor) + if ( + wantsrealtime and + hardware.get_emulator_thread_policy_constraint(flavor) == + fields.CPUEmulatorThreadsPolicy.SHARE and + not CONF.compute.cpu_shared_set + ): + # NOTE(stephenfin) Yes, it's horrible that we're doing this here, + # but the shared policy unfortunately has different behavior + # depending on whether the '[compute] cpu_shared_set' is configured + # or not and we need it to be configured. Also note that we have + # already handled other conditions, such as no emulator thread + # policy being configured whatsoever, at the API level. + LOG.warning( + 'Instance is requesting real-time CPUs with pooled ' + 'emulator threads, but a shared CPU pool has not been ' + 'configured on this host.' + ) + raise exception.RealtimeMaskNotFoundOrInvalid() + wantsmempages = False if inst_topology: for cell in inst_topology.cells: @@ -5439,8 +5459,6 @@ class LibvirtDriver(driver.ComputeDriver): wantsmempages = True break - wantsrealtime = hardware.is_realtime_enabled(flavor) - wantsfilebacked = CONF.libvirt.file_backed_memory > 0 if wantsmempages and wantsfilebacked: diff --git a/releasenotes/notes/bug-1884231-16acf297d88b122e.yaml b/releasenotes/notes/bug-1884231-16acf297d88b122e.yaml index a9096b9332f2..4f18a2fcb8b2 100644 --- a/releasenotes/notes/bug-1884231-16acf297d88b122e.yaml +++ b/releasenotes/notes/bug-1884231-16acf297d88b122e.yaml @@ -4,3 +4,8 @@ fixes: Previously, it was possible to specify values for the ``hw:cpu_realtime_mask`` extra spec that were not within the range of valid instances cores. This value is now correctly validated. +features: + - | + It is now possible to allocate all cores in an instance to realtime and + omit the ``hw:cpu_realtime_mask`` extra spec. This requires specifying the + ``hw:emulator_threads_policy`` extra spec.