openstack/nova
Code Issues Proposed changes

objects: Introduce the 'CPUAllocationPolicy.MIXED' enum

Browse Source 
Introduce a 'mixed' instance CPU allocation policy and
will be worked with upcoming patches, for purpose of
creating an instance combined shared CPUs with dedicated
or realtime CPUs.

In an instance mixed with different type of CPUs, the shared CPU
shared CPU time slots with other instances, and also might be a
CPU with less or un-guaranteed hardware resources, which implies
to have no guarantee for the behavior of the workload running on
it. If we call the shared CPU as 'low priority' CPU, then the
realtime or dedicated CPU could be called as 'high priority' CPU,
user could assign more hardware CPU resources or place some
guaranteed resource to it to let the workload to entail high
performance or stable service quality.

Based on https://review.opendev.org/714704

Part of blueprint use-pcpu-and-vcpu-in-one-instance

Change-Id: I99cfee14bb105a8792651129426c0c5a3749796d
Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com>
This commit is contained in:
Wang Huaqiang 2020-05-14 15:55:24 +08:00
parent d992eb2f5a
commit ba3388d666
17 changed files with 203 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
doc/notification_samples/common_payloads/ImageMetaPropsPayload.json
View File

@ -4,5 +4,5 @@
"hw_architecture": "x86_64"
},
"nova_object.name": "ImageMetaPropsPayload",
"nova_object.version": "1.3"
"nova_object.version": "1.4"
}

2
doc/notification_samples/common_payloads/InstanceNUMACellPayload.json
View File

@ -1,5 +1,5 @@
{
"nova_object.version": "1.1",
"nova_object.version": "1.2",
"nova_object.namespace": "nova",
"nova_object.name": "InstanceNUMACellPayload",
"nova_object.data": {

2
nova/api/openstack/compute/servers.py
View File

@ -87,6 +87,8 @@ INVALID_FLAVOR_IMAGE_EXCEPTIONS = (
exception.PciRequestAliasNotDefined,
exception.RealtimeConfigurationInvalid,
exception.RealtimeMaskNotFoundOrInvalid,
exception.RequiredMixedInstancePolicy,
exception.RequiredMixedOrRealtimeCPUMask,
)
MIN_COMPUTE_MOVE_BANDWIDTH = 39

7
nova/api/validation/extra_specs/hw.py
View File

@ -63,14 +63,17 @@ cpu_policy_validators = [
'CPUs can run on. If ``shared`` (default), guest CPUs can be '
'overallocated but cannot float across host cores. If '
'``dedicated``, guest CPUs cannot be overallocated but are '
'individually pinned to their own host core.'
'individually pinned to their own host core. ``mixed`` is a '
'policy with which the guest is mixing the overallocated and '
'pinned guest CPUs.'
),
value={
'type': str,
'description': 'The CPU policy.',
'enum': [
'dedicated',
'shared'
'shared',
'mixed',
],
},
),

11
nova/exception.py
View File

@ -2314,3 +2314,14 @@ class AcceleratorRequestOpFailed(NovaException):
class InvalidLibvirtGPUConfig(NovaException):
msg_fmt = _('Invalid configuration for GPU devices: %(reason)s')
class RequiredMixedInstancePolicy(Invalid):
msg_fmt = _("Cannot specify 'hw:cpu_dedicated_mask' without the "
"'mixed' policy.")
class RequiredMixedOrRealtimeCPUMask(Invalid):
msg_fmt = _("Must specify either 'hw:cpu_dedicated_mask' or "
"'hw:cpu_realtime_mask' when using 'mixed' CPU policy"
" instance.")

3
nova/notifications/objects/image.py
View File

@ -120,7 +120,8 @@ class ImageMetaPropsPayload(base.NotificationPayloadBase):
# Version 1.1: Added 'gop', 'virtio' and 'none' to hw_video_model field
# Version 1.2: Added hw_pci_numa_affinity_policy field
# Version 1.3: Added hw_mem_encryption, hw_pmu and hw_time_hpet fields
VERSION = '1.3'
# Version 1.4: Added 'mixed' to hw_cpu_policy field
VERSION = '1.4'
SCHEMA = {
k: ('image_meta_props', k) for k in image_meta.ImageMetaProps.fields}

3
nova/notifications/objects/request_spec.py
View File

@ -144,7 +144,8 @@ class InstanceNUMATopologyPayload(base.NotificationPayloadBase):
class InstanceNUMACellPayload(base.NotificationPayloadBase):
# Version 1.0: Initial version
# Version 1.1: Added pcpuset field
VERSION = '1.1'
# Version 1.2: Added 'mixed' to cpu_policy field
VERSION = '1.2'
SCHEMA = {
'id': ('numa_cell', 'id'),

3
nova/objects/fields.py
View File

@ -272,8 +272,9 @@ class CPUAllocationPolicy(BaseNovaEnum):
DEDICATED = "dedicated"
SHARED = "shared"
MIXED = "mixed"
ALL = (DEDICATED, SHARED)
ALL = (DEDICATED, SHARED, MIXED)
class CPUThreadAllocationPolicy(BaseNovaEnum):

10
nova/objects/image_meta.py
View File

@ -175,14 +175,22 @@ class ImageMetaProps(base.NovaObject):
# Version 1.23: Added 'hw_pmu' field
# Version 1.24: Added 'hw_mem_encryption' field
# Version 1.25: Added 'hw_pci_numa_affinity_policy' field
# Version 1.26: Added 'mixed' to 'hw_cpu_policy' field
# NOTE(efried): When bumping this version, the version of
# ImageMetaPropsPayload must also be bumped. See its docstring for details.
VERSION = '1.25'
VERSION = '1.26'
def obj_make_compatible(self, primitive, target_version):
super(ImageMetaProps, self).obj_make_compatible(primitive,
target_version)
target_version = versionutils.convert_version_to_tuple(target_version)
if target_version < (1, 26):
policy = primitive.get('hw_cpu_policy', None)
if policy == fields.CPUAllocationPolicy.MIXED:
raise exception.ObjectActionError(
action='obj_make_compatible',
reason='hw_cpu_policy=%s not supported in version %s' %
(policy, target_version))
if target_version < (1, 25):
primitive.pop('hw_pci_numa_affinity_policy', None)
if target_version < (1, 24):

13
nova/objects/instance_numa.py
View File

@ -19,6 +19,7 @@ from oslo_utils import versionutils
from nova.db import api as db
from nova import exception
from nova.i18n import _
from nova.objects import base
from nova.objects import fields as obj_fields
from nova.virt import hardware
@ -34,12 +35,22 @@ class InstanceNUMACell(base.NovaEphemeralObject,
# Version 1.3: Add cpu_policy and cpu_thread_policy fields
# Version 1.4: Add cpuset_reserved field
# Version 1.5: Add pcpuset field
VERSION = '1.5'
# Version 1.6: Add 'mixed' to cpu_policy field
VERSION = '1.6'
def obj_make_compatible(self, primitive, target_version):
super(InstanceNUMACell, self).obj_make_compatible(primitive,
target_version)
target_version = versionutils.convert_version_to_tuple(target_version)
# Instance with a 'mixed' CPU policy could not provide a backward
# compatibility.
if target_version < (1, 6):
if primitive['cpu_policy'] == obj_fields.CPUAllocationPolicy.MIXED:
raise exception.ObjectActionError(
action='obj_make_compatible',
reason=_('mixed instance is not supported in version %s') %
target_version)
# NOTE(huaqiang): Since version 1.5, 'cpuset' is modified to track the
# unpinned CPUs only, with pinned CPUs tracked via 'pcpuset' instead.
# For a backward compatibility, move the 'dedicated' instance CPU list

4
nova/tests/functional/notification_sample_tests/test_instance.py
View File

@ -1249,7 +1249,7 @@ class TestInstanceNotificationSample(
'nova_object.data': {},
'nova_object.name': 'ImageMetaPropsPayload',
'nova_object.namespace': 'nova',
'nova_object.version': u'1.3'},
'nova_object.version': u'1.4'},
'image.size': 58145823,
'image.tags': [],
'scheduler_hints': {'_nova_check_type': ['rebuild']},
@ -1344,7 +1344,7 @@ class TestInstanceNotificationSample(
'nova_object.data': {},
'nova_object.name': 'ImageMetaPropsPayload',
'nova_object.namespace': 'nova',
'nova_object.version': u'1.3'},
'nova_object.version': u'1.4'},
'image.size': 58145823,
'image.tags': [],
'scheduler_hints': {'_nova_check_type': ['rebuild']},

1
nova/tests/unit/api/validation/extra_specs/test_validators.py
View File

@ -69,6 +69,7 @@ class TestValidators(test.NoDBTestCase):
('hw:cpu_thread_policy', 'prefer'),
('hw:emulator_threads_policy', 'isolate'),
('hw:pci_numa_affinity_policy', 'legacy'),
('hw:cpu_policy', 'mixed'),
)
for key, value in valid_specs:
validators.validate(key, value)

9
nova/tests/unit/compute/test_compute.py
View File

@ -13352,10 +13352,17 @@ class CheckRequestedImageTestCase(test.TestCase):
self.context, image['id'],
image, self.instance_type, root_bdm)
def test_cpu_policy(self):
@mock.patch('nova.virt.hardware.get_dedicated_cpu_constraint')
def test_cpu_policy(self, dedicated_cpu_mock):
image = {'id': uuids.image_id, 'status': 'active'}
for v in obj_fields.CPUAllocationPolicy.ALL:
image['properties'] = {'hw_cpu_policy': v}
# 'mixed' policy requires a definition of 'cpu_dedicated_mask'
if v == obj_fields.CPUAllocationPolicy.MIXED:
dedicated_cpu_mock.return_value = set([0])
else:
dedicated_cpu_mock.return_value = None
self.compute_api._validate_flavor_image(
self.context, image['id'], image, self.instance_type, None)
image['properties'] = {'hw_cpu_policy': 'bar'}

4
nova/tests/unit/notifications/objects/test_notification.py
View File

@ -387,7 +387,7 @@ notification_object_data = {
# ImageMetaProps, so when you see a fail here for that reason, you must
# *also* bump the version of ImageMetaPropsPayload. See its docstring for
# more information.
'ImageMetaPropsPayload': '1.3-9c200c895932163a4e14e6bb385fa1e0',
'ImageMetaPropsPayload': '1.4-036c794843b95a3a39ee70830f5f6557',
'InstanceActionNotification': '1.0-a73147b93b520ff0061865849d3dfa56',
'InstanceActionPayload': '1.8-4fa3da9cbf0761f1f700ae578f36dc2f',
'InstanceActionRebuildNotification':
@ -411,7 +411,7 @@ notification_object_data = {
'InstanceActionSnapshotPayload': '1.9-c3e0bbaaefafdfa2f8e6e504c2c9b12c',
'InstanceExistsNotification': '1.0-a73147b93b520ff0061865849d3dfa56',
'InstanceExistsPayload': '1.2-e082c02438ee57164829afaeee3bf7f8',
'InstanceNUMACellPayload': '1.1-2a24ab42bf5e8dfa98291402725bf278',
'InstanceNUMACellPayload': '1.2-a367add3378c71c21c817ab2b23db3bf',
'InstanceNUMATopologyPayload': '1.0-247361b152047c18ae9ad1da2544a3c9',
'InstancePCIRequestPayload': '1.0-12d0d61baf183daaafd93cbeeed2956f',
'InstancePCIRequestsPayload': '1.0-6751cffe0c0fabd212aad624f672429a',

4
nova/tests/unit/objects/test_objects.py
View File

@ -1077,7 +1077,7 @@ object_data = {
'HyperVLiveMigrateData': '1.4-e265780e6acfa631476c8170e8d6fce0',
'IDEDeviceBus': '1.0-29d4c9f27ac44197f01b6ac1b7e16502',
'ImageMeta': '1.8-642d1b2eb3e880a367f37d72dd76162d',
'ImageMetaProps': '1.25-66fc973af215eb5701ed4034bb6f0685',
'ImageMetaProps': '1.26-b9f136cd10a2b5ffb3ae44332f2f687d',
'Instance': '2.7-d187aec68cad2e4d8b8a03a68e4739ce',
'InstanceAction': '1.2-9a5abc87fdd3af46f45731960651efb5',
'InstanceActionEvent': '1.4-5b1f361bd81989f8bb2c20bb7e8a4cb4',
@ -1093,7 +1093,7 @@ object_data = {
'InstanceList': '2.6-238f125650c25d6d12722340d726f723',
'InstanceMapping': '1.2-3bd375e65c8eb9c45498d2f87b882e03',
'InstanceMappingList': '1.3-d34b6ebb076d542ae0f8b440534118da',
'InstanceNUMACell': '1.5-d6f884326eba8cae60930e06047fc7d9',
'InstanceNUMACell': '1.6-25d9120d83a18356f4146f2a6fe2cc8d',
'InstanceNUMATopology': '1.3-ec0030cb0402a49c96da7051c037082a',
'InstancePCIRequest': '1.3-f6d324f1c337fad4f34892ed5f484c9a',
'InstancePCIRequests': '1.1-65e38083177726d806684cb1cc0136d2',

87
nova/tests/unit/virt/test_hardware.py
View File

@ -932,10 +932,75 @@ class NUMATopologyTest(test.NoDBTestCase):
),
"image": {
"properties": {
"hw_cpu_policy": "shared"
}
},
"expect": fields.CPUAllocationPolicy.DEDICATED
},
{
"flavor": objects.Flavor(
extra_specs={
"hw:cpu_policy": "dedicated"
}
),
"image": {
"properties": {
}
},
"expect": fields.CPUAllocationPolicy.DEDICATED
},
{
"flavor": objects.Flavor(
extra_specs={
"hw:cpu_policy": "mixed"
}
),
"image": {
"properties": {
"hw_cpu_policy": "dedicated"
}
},
"expect": exception.ImageCPUPinningForbidden
},
{
"flavor": objects.Flavor(
extra_specs={
"hw:cpu_policy": "mixed"
}
),
"image": {
"properties": {
"hw_cpu_policy": "mixed"
}
},
"expect": fields.CPUAllocationPolicy.MIXED
},
{
"flavor": objects.Flavor(
extra_specs={
"hw:cpu_policy": "mixed"
}
),
"image": {
"properties": {
"hw_cpu_policy": "shared"
}
},
"expect": fields.CPUAllocationPolicy.MIXED
},
{
"flavor": objects.Flavor(
extra_specs={
"hw:cpu_policy": "mixed"
}
),
"image": {
"properties": {
}
},
"expect": fields.CPUAllocationPolicy.MIXED
},
{
"flavor": objects.Flavor(
extra_specs={
@ -949,6 +1014,19 @@ class NUMATopologyTest(test.NoDBTestCase):
},
"expect": exception.ImageCPUPinningForbidden
},
{
"flavor": objects.Flavor(
extra_specs={
"hw:cpu_policy": "shared"
}
),
"image": {
"properties": {
"hw_cpu_policy": "mixed"
}
},
"expect": exception.ImageCPUPinningForbidden
},
{
"flavor": objects.Flavor(
extra_specs={
@ -983,6 +1061,15 @@ class NUMATopologyTest(test.NoDBTestCase):
},
"expect": fields.CPUAllocationPolicy.DEDICATED
},
{
"flavor": objects.Flavor(),
"image": {
"properties": {
"hw_cpu_policy": "mixed"
}
},
"expect": fields.CPUAllocationPolicy.MIXED
},
{
"flavor": objects.Flavor(),
"image": {

58
nova/virt/hardware.py
View File

@ -1505,10 +1505,17 @@ def get_cpu_policy_constraint(
if flavor_policy == fields.CPUAllocationPolicy.DEDICATED:
cpu_policy = flavor_policy
elif flavor_policy == fields.CPUAllocationPolicy.SHARED:
elif flavor_policy == fields.CPUAllocationPolicy.MIXED:
if image_policy == fields.CPUAllocationPolicy.DEDICATED:
raise exception.ImageCPUPinningForbidden()
cpu_policy = flavor_policy
elif flavor_policy == fields.CPUAllocationPolicy.SHARED:
if image_policy in (
fields.CPUAllocationPolicy.MIXED,
fields.CPUAllocationPolicy.DEDICATED,
):
raise exception.ImageCPUPinningForbidden()
cpu_policy = flavor_policy
elif image_policy in fields.CPUAllocationPolicy.ALL:
cpu_policy = image_policy
else:
@ -1693,6 +1700,21 @@ def _get_hyperthreading_trait(
return None
# NOTE(stephenfin): This must be public as it's used elsewhere
# TODO(Huaqiang): To be filled with the logic of parsing
# 'hw:cpu_dedicated_mask' and relevant test cases in later patches once the
# code is ready to build up an instance in 'mixed' CPU allocation policy.
def get_dedicated_cpu_constraint(
flavor: 'objects.Flavor',
) -> ty.Optional[ty.Set[int]]:
"""Validate and return the requested dedicated CPU mask.
:param flavor: ``nova.objects.Flavor`` instance
:returns: The dedicated CPUs requested, else None.
"""
return None
# NOTE(stephenfin): This must be public as it's used elsewhere
def get_realtime_cpu_constraint(
flavor: 'objects.Flavor',
@ -1833,12 +1855,18 @@ def numa_get_constraints(flavor, image_meta):
with invalid value in image or flavor.
:raises: exception.InvalidRequest if there is a conflict between explicitly
and implicitly requested resources of hyperthreading traits
:raises: exception.RequiredMixedInstancePolicy if dedicated CPU mask is
provided in flavor while CPU policy is not 'mixed'.
:raises: exception.RequiredMixedOrRealtimeCPUMask the mixed policy instance
dedicated CPU mask can only be specified through either
'hw:cpu_realtime_mask' or 'hw:cpu_dedicated_mask', not both.
:returns: objects.InstanceNUMATopology, or None
"""
cpu_policy = get_cpu_policy_constraint(flavor, image_meta)
cpu_thread_policy = get_cpu_thread_policy_constraint(flavor, image_meta)
rt_mask = get_realtime_cpu_constraint(flavor, image_meta)
dedicated_cpus = get_dedicated_cpu_constraint(flavor)
emu_threads_policy = get_emulator_thread_policy_constraint(flavor)
# handle explicit VCPU/PCPU resource requests and the HW_CPU_HYPERTHREADING
@ -1904,13 +1932,36 @@ def numa_get_constraints(flavor, image_meta):
if emu_threads_policy == fields.CPUEmulatorThreadsPolicy.ISOLATE:
raise exception.BadRequirementEmulatorThreadsPolicy()
# 'hw:cpu_dedicated_mask' should not be defined in a flavor with
# 'shared' policy.
if dedicated_cpus:
raise exception.RequiredMixedInstancePolicy()
if rt_mask:
raise exception.RealtimeConfigurationInvalid()
elif cpu_policy == fields.CPUAllocationPolicy.DEDICATED:
# 'hw:cpu_dedicated_mask' should not be defined in a flavor with
# 'dedicated' policy.
if dedicated_cpus:
raise exception.RequiredMixedInstancePolicy()
# But for an instance with 'dedicated' CPU allocation policy, all
# CPUs are 'dedicated' CPUs, which is 1:1 pinned to a host CPU.
dedicated_cpus = set(range(flavor.vcpus))
else: # MIXED
# FIXME(huaqiang): So far, 'mixed' instance is not supported
# and the 'dedicated_cpus' variable is set to 'None' due to being not
# ready to parse 'hw:cpu_dedicated_mask'.
# The logic of parsing 'hw:cpu_dedicated_mask' should be added once
# the code is ready for setting up an 'mixed' instance.
if dedicated_cpus is None:
raise exception.RequiredMixedOrRealtimeCPUMask()
nodes = _get_numa_node_count_constraint(flavor, image_meta)
pagesize = _get_numa_pagesize_constraint(flavor, image_meta)
vpmems = get_vpmems(flavor)
dedicated_cpus = dedicated_cpus or set()
# NOTE(stephenfin): There are currently four things that will configure a
# NUMA topology for an instance:
#
@ -1920,14 +1971,13 @@ def numa_get_constraints(flavor, image_meta):
# - The use of vPMEM
if nodes or pagesize or vpmems or cpu_policy in (
fields.CPUAllocationPolicy.DEDICATED,
fields.CPUAllocationPolicy.MIXED,
):
# NOTE(huaqiang): Here we build the instance dedicated CPU set and the
# shared CPU set, through 'pcpus' and 'vcpus' respectively,
# which will be used later to calculate the per-NUMA-cell CPU set.
cpus = set(range(flavor.vcpus))
pcpus = set()
if cpu_policy == fields.CPUAllocationPolicy.DEDICATED:
pcpus = cpus
pcpus = dedicated_cpus
vcpus = cpus - pcpus
nodes = nodes or 1