From 8c7224172641c6194582ca4cf7ce11e907df50aa Mon Sep 17 00:00:00 2001 From: Sean Mooney Date: Thu, 1 Aug 2019 15:00:07 +0000 Subject: [PATCH] support pci numa affinity policies in flavor and image This addresses bug #1795920 by adding support for defining a pci numa affinity policy via the flavor extra specs or image metadata properties enabling the policies to be applied to neutron sriov port including hardware offloaded ovs. Closes-Bug: #1795920 Related-Bug: #1805891 Implements: blueprint vm-scoped-sriov-numa-affinity Change-Id: Ibd62b24c2bd2dd208d0f804378d4e4f2bbfdaed6 --- .../ImageMetaPropsPayload.json | 2 +- doc/source/admin/pci-passthrough.rst | 3 +- doc/source/user/flavors.rst | 36 ++++++ nova/compute/api.py | 8 +- nova/exception.py | 10 ++ nova/network/api.py | 7 +- nova/network/neutronv2/api.py | 9 +- nova/notifications/objects/image.py | 6 +- nova/objects/image_meta.py | 8 +- nova/pci/request.py | 11 +- .../libvirt/test_pci_sriov_servers.py | 120 ++++++++++++++++++ .../test_instance.py | 4 +- nova/tests/unit/api/openstack/fakes.py | 5 +- .../objects/test_notification.py | 2 +- nova/tests/unit/objects/test_objects.py | 2 +- nova/tests/unit/pci/test_request.py | 60 +++++++++ nova/tests/unit/virt/libvirt/fakelibvirt.py | 5 + nova/tests/unit/virt/test_hardware.py | 71 +++++++++++ nova/virt/hardware.py | 22 ++++ ...numa-affinity-policy-b49858452827c727.yaml | 13 ++ 20 files changed, 384 insertions(+), 20 deletions(-) create mode 100644 releasenotes/notes/sriov-numa-affinity-policy-b49858452827c727.yaml diff --git a/doc/notification_samples/common_payloads/ImageMetaPropsPayload.json b/doc/notification_samples/common_payloads/ImageMetaPropsPayload.json index 2dcc72ba4534..dac82f92d4e0 100644 --- a/doc/notification_samples/common_payloads/ImageMetaPropsPayload.json +++ b/doc/notification_samples/common_payloads/ImageMetaPropsPayload.json @@ -4,5 +4,5 @@ "hw_architecture": "x86_64" }, "nova_object.name": "ImageMetaPropsPayload", - "nova_object.version": "1.1" + "nova_object.version": "1.2" } diff --git a/doc/source/admin/pci-passthrough.rst b/doc/source/admin/pci-passthrough.rst index 48abf1b2032e..663fdbaf518f 100644 --- a/doc/source/admin/pci-passthrough.rst +++ b/doc/source/admin/pci-passthrough.rst @@ -171,9 +171,10 @@ found on the compute nodes. For example: .. code-block:: ini [pci] - alias = { "vendor_id":"8086", "product_id":"154d", "device_type":"type-PF", "name":"a1" } + alias = { "vendor_id":"8086", "product_id":"154d", "device_type":"type-PF", "name":"a1", "numa_policy":"preferred" } Refer to :oslo.config:option:`pci.alias` for syntax information. +Refer to :ref:`Affinity ` for ``numa_policy`` information. Once configured, restart the :program:`nova-api` service. diff --git a/doc/source/user/flavors.rst b/doc/source/user/flavors.rst index f8d2e652d078..972d48e8d744 100644 --- a/doc/source/user/flavors.rst +++ b/doc/source/user/flavors.rst @@ -480,6 +480,42 @@ CPU pinning policy The ``hw:cpu_thread_policy`` option is only valid if ``hw:cpu_policy`` is set to ``dedicated``. +.. _pci_numa_affinity_policy: + +PCI NUMA Affinity Policy + For the libvirt driver, you can specify the NUMA affinity policy for + PCI passthrough devices and neutron SR-IOV interfaces via the + ``hw:pci_numa_affinity_policy`` flavor extra spec or + ``hw_pci_numa_affinity_policy`` image property. The allowed values are + ``required``,``preferred`` or ``legacy`` (default). + + **required** + This value will mean that nova will boot instances with PCI devices + **only** if at least one of the NUMA nodes of the instance is associated + with these PCI devices. It means that if NUMA node info for some PCI + devices could not be determined, those PCI devices wouldn't be consumable + by the instance. This provides maximum performance. + + **preferred** + This value will mean that ``nova-scheduler`` will choose a compute host + with minimal consideration for the NUMA affinity of PCI devices. + ``nova-compute`` will attempt a best effort selection of PCI devices + based on NUMA affinity, however, if this is not possible then + ``nova-compute`` will fall back to scheduling on a NUMA node that is not + associated with the PCI device. + + **legacy** + This is the default value and it describes the current nova behavior. + Usually we have information about association of PCI devices with NUMA + nodes. However, some PCI devices do not provide such information. The + ``legacy`` value will mean that nova will boot instances with PCI device + if either: + + * The PCI device is associated with at least one NUMA nodes on which the + instance will be booted + + * There is no information about PCI-NUMA affinity available + .. _extra-specs-numa-topology: NUMA topology diff --git a/nova/compute/api.py b/nova/compute/api.py index 11b782a53935..1aab75050ae2 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -915,6 +915,9 @@ class API(base.Base): system_metadata = {} + pci_numa_affinity_policy = hardware.get_pci_numa_policy_constraint( + instance_type, image_meta) + # PCI requests come from two sources: instance flavor and # requested_networks. The first call in below returns an # InstancePCIRequests object which is a list of InstancePCIRequest @@ -922,9 +925,10 @@ class API(base.Base): # object for each SR-IOV port, and append it to the list in the # InstancePCIRequests object pci_request_info = pci_request.get_pci_requests_from_flavor( - instance_type) + instance_type, affinity_policy=pci_numa_affinity_policy) result = self.network_api.create_resource_requests( - context, requested_networks, pci_request_info) + context, requested_networks, pci_request_info, + affinity_policy=pci_numa_affinity_policy) network_metadata, port_resource_requests = result # Creating servers with ports that have resource requests, like QoS diff --git a/nova/exception.py b/nova/exception.py index d8ee0de665bb..d0fd0fe8a207 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -1861,6 +1861,12 @@ class ImageNUMATopologyRebuildConflict(Invalid): "The image provided is invalid for this instance.") +class ImagePCINUMAPolicyForbidden(Forbidden): + msg_fmt = _("Image property 'hw_pci_numa_affinity_policy' is not " + "permitted to override the 'hw:pci_numa_affinity_policy' " + "flavor extra spec.") + + class ImageNUMATopologyAsymmetric(Invalid): msg_fmt = _("Instance CPUs and/or memory cannot be evenly distributed " "across instance NUMA nodes. Explicit assignment of CPUs " @@ -2254,6 +2260,10 @@ class InvalidNetworkNUMAAffinity(Invalid): msg_fmt = _("Invalid NUMA network affinity configured: %(reason)s") +class InvalidPCINUMAAffinity(Invalid): + msg_fmt = _("Invalid PCI NUMA affinity configured: %(policy)s") + + class PowerVMAPIFailed(NovaException): msg_fmt = _("PowerVM API failed to complete for instance=%(inst_name)s. " "%(reason)s") diff --git a/nova/network/api.py b/nova/network/api.py index b37be81e7e3b..ba2ef9ea14e4 100644 --- a/nova/network/api.py +++ b/nova/network/api.py @@ -355,8 +355,9 @@ class API(base_api.NetworkAPI): # the requested number in this case. return num_instances - def create_resource_requests(self, context, requested_networks, - pci_requests=None): + def create_resource_requests( + self, context, requested_networks, pci_requests=None, + affinity_policy=None): """Retrieve all information for the networks passed at the time of creating the server. @@ -366,6 +367,8 @@ class API(base_api.NetworkAPI): :param pci_requests: The list of PCI requests to which additional PCI requests created here will be added. :type pci_requests: nova.objects.InstancePCIRequests + :param affinity_policy: requested pci numa affinity policy + :type affinity_policy: nova.objects.fields.PCINUMAAffinityPolicy :returns: A tuple with an instance of ``objects.NetworkMetadata`` for use by the scheduler or None and a list of RequestGroup diff --git a/nova/network/neutronv2/api.py b/nova/network/neutronv2/api.py index e2b8a6083ed3..d8f361eefb92 100644 --- a/nova/network/neutronv2/api.py +++ b/nova/network/neutronv2/api.py @@ -1971,8 +1971,9 @@ class API(base_api.NetworkAPI): resource_request = port.get(constants.RESOURCE_REQUEST, None) return vnic_type, trusted, network_id, resource_request - def create_resource_requests(self, context, requested_networks, - pci_requests=None): + def create_resource_requests( + self, context, requested_networks, pci_requests=None, + affinity_policy=None): """Retrieve all information for the networks passed at the time of creating the server. @@ -1982,6 +1983,8 @@ class API(base_api.NetworkAPI): :param pci_requests: The list of PCI requests to which additional PCI requests created here will be added. :type pci_requests: nova.objects.InstancePCIRequests + :param affinity_policy: requested pci numa affinity policy + :type affinity_policy: nova.objects.fields.PCINUMAAffinityPolicy :returns: A tuple with an instance of ``objects.NetworkMetadata`` for use by the scheduler or None and a list of RequestGroup @@ -2062,6 +2065,8 @@ class API(base_api.NetworkAPI): spec=[spec], request_id=uuidutils.generate_uuid(), requester_id=requester_id) + if affinity_policy: + request.numa_policy = affinity_policy pci_requests.requests.append(request) pci_request_id = request.request_id diff --git a/nova/notifications/objects/image.py b/nova/notifications/objects/image.py index b8b75a5abc70..3971a5f1b0ea 100644 --- a/nova/notifications/objects/image.py +++ b/nova/notifications/objects/image.py @@ -106,7 +106,8 @@ class ImageMetaPayload(base.NotificationPayloadBase): class ImageMetaPropsPayload(base.NotificationPayloadBase): # Version 1.0: Initial version # Version 1.1: Added 'gop', 'virtio' and 'none' to hw_video_model field - VERSION = '1.1' + # Version 1.2: Added hw_pci_numa_affinity_policy field + VERSION = '1.2' SCHEMA = { 'hw_architecture': ('image_meta_props', 'hw_architecture'), @@ -133,6 +134,8 @@ class ImageMetaPropsPayload(base.NotificationPayloadBase): 'hw_numa_nodes': ('image_meta_props', 'hw_numa_nodes'), 'hw_numa_cpus': ('image_meta_props', 'hw_numa_cpus'), 'hw_numa_mem': ('image_meta_props', 'hw_numa_mem'), + 'hw_pci_numa_affinity_policy': ('image_meta_props', + 'hw_pci_numa_affinity_policy'), 'hw_pointer_model': ('image_meta_props', 'hw_pointer_model'), 'hw_qemu_guest_agent': ('image_meta_props', 'hw_qemu_guest_agent'), 'hw_rescue_bus': ('image_meta_props', 'hw_rescue_bus'), @@ -210,6 +213,7 @@ class ImageMetaPropsPayload(base.NotificationPayloadBase): 'hw_numa_nodes': fields.IntegerField(), 'hw_numa_cpus': fields.ListOfSetsOfIntegersField(), 'hw_numa_mem': fields.ListOfIntegersField(), + 'hw_pci_numa_affinity_policy': fields.PCINUMAAffinityPolicyField(), 'hw_pointer_model': fields.PointerModelField(), 'hw_qemu_guest_agent': fields.FlexibleBooleanField(), 'hw_rescue_bus': fields.DiskBusField(), diff --git a/nova/objects/image_meta.py b/nova/objects/image_meta.py index f81812f1fdc7..de28041276a2 100644 --- a/nova/objects/image_meta.py +++ b/nova/objects/image_meta.py @@ -174,12 +174,15 @@ class ImageMetaProps(base.NovaObject): # Version 1.22: Added 'gop', 'virtio' and 'none' to hw_video_model field # Version 1.23: Added 'hw_pmu' field # Version 1.24: Added 'hw_mem_encryption' field - VERSION = '1.24' + # Version 1.25: Added 'hw_pci_numa_affinity_policy' field + VERSION = '1.25' def obj_make_compatible(self, primitive, target_version): super(ImageMetaProps, self).obj_make_compatible(primitive, target_version) target_version = versionutils.convert_version_to_tuple(target_version) + if target_version < (1, 25): + primitive.pop('hw_pci_numa_affinity_policy', None) if target_version < (1, 24): primitive.pop('hw_mem_encryption', None) if target_version < (1, 23): @@ -334,6 +337,9 @@ class ImageMetaProps(base.NovaObject): # list value indicates the memory size of that node. 'hw_numa_mem': fields.ListOfIntegersField(), + # Enum field to specify pci device NUMA affinity. + 'hw_pci_numa_affinity_policy': fields.PCINUMAAffinityPolicyField(), + # Generic property to specify the pointer model type. 'hw_pointer_model': fields.PointerModelField(), diff --git a/nova/pci/request.py b/nova/pci/request.py index 845a3d52bbb9..4857be227955 100644 --- a/nova/pci/request.py +++ b/nova/pci/request.py @@ -148,7 +148,7 @@ def _get_alias_from_config(): return aliases -def _translate_alias_to_requests(alias_spec): +def _translate_alias_to_requests(alias_spec, affinity_policy=None): """Generate complete pci requests from pci aliases in extra_spec.""" pci_aliases = _get_alias_from_config() @@ -160,6 +160,7 @@ def _translate_alias_to_requests(alias_spec): count = int(count) numa_policy, spec = pci_aliases[name] + policy = affinity_policy or numa_policy # NOTE(gibi): InstancePCIRequest has a requester_id field that could # be filled with the flavor.flavorid but currently there is no special @@ -169,7 +170,7 @@ def _translate_alias_to_requests(alias_spec): count=count, spec=spec, alias_name=name, - numa_policy=numa_policy)) + numa_policy=policy)) return pci_requests @@ -227,7 +228,7 @@ def get_instance_pci_request_from_vif(context, instance, vif): node_id=cn_id) -def get_pci_requests_from_flavor(flavor): +def get_pci_requests_from_flavor(flavor, affinity_policy=None): """Validate and return PCI requests. The ``pci_passthrough:alias`` extra spec describes the flavor's PCI @@ -265,6 +266,7 @@ def get_pci_requests_from_flavor(flavor): }] :param flavor: The flavor to be checked + :param affinity_policy: pci numa affinity policy :returns: A list of PCI requests :rtype: nova.objects.InstancePCIRequests :raises: exception.PciRequestAliasNotDefined if an invalid PCI alias is @@ -276,6 +278,7 @@ def get_pci_requests_from_flavor(flavor): if ('extra_specs' in flavor and 'pci_passthrough:alias' in flavor['extra_specs']): pci_requests = _translate_alias_to_requests( - flavor['extra_specs']['pci_passthrough:alias']) + flavor['extra_specs']['pci_passthrough:alias'], + affinity_policy=affinity_policy) return objects.InstancePCIRequests(requests=pci_requests) diff --git a/nova/tests/functional/libvirt/test_pci_sriov_servers.py b/nova/tests/functional/libvirt/test_pci_sriov_servers.py index f4cef3528e93..b302366e9f6e 100644 --- a/nova/tests/functional/libvirt/test_pci_sriov_servers.py +++ b/nova/tests/functional/libvirt/test_pci_sriov_servers.py @@ -13,8 +13,10 @@ # License for the specific language governing permissions and limitations # under the License. +import ddt import fixtures import mock + from oslo_log import log as logging from oslo_serialization import jsonutils @@ -397,3 +399,121 @@ class PCIServersWithRequiredNUMATest(PCIServersWithPreferredNUMATest): } )] end_status = 'ERROR' + + +@ddt.ddt +class PCIServersWithSRIOVAffinityPoliciesTest(_PCIServersTestBase): + + # The order of the filters is required to make the assertion that the + # PciPassthroughFilter is invoked in _run_build_test pass in the + # numa affinity tests otherwise the NUMATopologyFilter will eliminate + # all hosts before we execute the PciPassthroughFilter. + ADDITIONAL_FILTERS = ['PciPassthroughFilter', 'NUMATopologyFilter'] + ALIAS_NAME = 'a1' + PCI_PASSTHROUGH_WHITELIST = [jsonutils.dumps( + { + 'vendor_id': fakelibvirt.PCI_VEND_ID, + 'product_id': fakelibvirt.PCI_PROD_ID, + } + )] + # we set the numa_affinity policy to required to ensure strict affinity + # between pci devices and the guest cpu and memory will be enforced. + PCI_ALIAS = [jsonutils.dumps( + { + 'vendor_id': fakelibvirt.PCI_VEND_ID, + 'product_id': fakelibvirt.PCI_PROD_ID, + 'name': ALIAS_NAME, + 'device_type': fields.PciDeviceType.STANDARD, + 'numa_policy': fields.PCINUMAAffinityPolicy.REQUIRED, + } + )] + + # NOTE(sean-k-mooney): i could just apply the ddt decorators + # to this function for the most part but i have chosen to + # keep one top level function per policy to make documenting + # the test cases simpler. + def _test_policy(self, pci_numa_node, status, policy): + host_info = fakelibvirt.HostInfo(cpu_nodes=2, cpu_sockets=1, + cpu_cores=2, cpu_threads=2, + kB_mem=15740000) + pci_info = fakelibvirt.HostPCIDevicesInfo( + num_pci=1, numa_node=pci_numa_node) + fake_connection = self._get_connection(host_info, pci_info) + self.mock_conn.return_value = fake_connection + + # only allow cpus on numa node 1 to be used for pinning + self.flags(cpu_dedicated_set='4-7', group='compute') + + # request cpu pinning to create a numa toplogy and allow the test to + # force which numa node the vm would have to be pinned too. + extra_spec = { + 'hw:cpu_policy': 'dedicated', + 'pci_passthrough:alias': '%s:1' % self.ALIAS_NAME, + 'hw:pci_numa_affinity_policy': policy + } + flavor_id = self._create_flavor(extra_spec=extra_spec) + self._run_build_test(flavor_id, end_status=status) + + @ddt.unpack # unpacks each sub-tuple e.g. *(pci_numa_node, status) + # the preferred policy should always pass regardless of numa affinity + @ddt.data((-1, 'ACTIVE'), (0, 'ACTIVE'), (1, 'ACTIVE')) + def test_create_server_with_sriov_numa_affinity_policy_preferred( + self, pci_numa_node, status): + """Validate behavior of 'preferred' PCI NUMA affinity policy. + + This test ensures that it *is* possible to allocate CPU and memory + resources from one NUMA node and a PCI device from another *if* + the SR-IOV NUMA affinity policy is set to preferred. + """ + self._test_policy(pci_numa_node, status, 'preferred') + + @ddt.unpack # unpacks each sub-tuple e.g. *(pci_numa_node, status) + # the legacy policy allow a PCI device to be used if it has NUMA + # affinity or if no NUMA info is available so we set the NUMA + # node for this device to -1 which is the sentinel value use by the + # Linux kernel for a device with no NUMA affinity. + @ddt.data((-1, 'ACTIVE'), (0, 'ERROR'), (1, 'ACTIVE')) + def test_create_server_with_sriov_numa_affinity_policy_legacy( + self, pci_numa_node, status): + """Validate behavior of 'legacy' PCI NUMA affinity policy. + + This test ensures that it *is* possible to allocate CPU and memory + resources from one NUMA node and a PCI device from another *if* + the SR-IOV NUMA affinity policy is set to legacy and the device + does not report NUMA information. + """ + self._test_policy(pci_numa_node, status, 'legacy') + + @ddt.unpack # unpacks each sub-tuple e.g. *(pci_numa_node, status) + # The required policy requires a PCI device to both report a NUMA + # and for the guest cpus and ram to be affinitized to the same + # NUMA node so we create 1 pci device in the first NUMA node. + @ddt.data((-1, 'ERROR'), (0, 'ERROR'), (1, 'ACTIVE')) + def test_create_server_with_sriov_numa_affinity_policy_required( + self, pci_numa_node, status): + """Validate behavior of 'required' PCI NUMA affinity policy. + + This test ensures that it *is not* possible to allocate CPU and memory + resources from one NUMA node and a PCI device from another *if* + the SR-IOV NUMA affinity policy is set to required and the device + does reports NUMA information. + """ + + # we set the numa_affinity policy to preferred to allow the PCI device + # to be selected from any numa node so we can prove the flavor + # overrides the alias. + alias = [jsonutils.dumps( + { + 'vendor_id': fakelibvirt.PCI_VEND_ID, + 'product_id': fakelibvirt.PCI_PROD_ID, + 'name': self.ALIAS_NAME, + 'device_type': fields.PciDeviceType.STANDARD, + 'numa_policy': fields.PCINUMAAffinityPolicy.PREFERRED, + } + )] + + self.flags(passthrough_whitelist=self.PCI_PASSTHROUGH_WHITELIST, + alias=alias, + group='pci') + + self._test_policy(pci_numa_node, status, 'required') diff --git a/nova/tests/functional/notification_sample_tests/test_instance.py b/nova/tests/functional/notification_sample_tests/test_instance.py index 641056efa474..c3d560019e53 100644 --- a/nova/tests/functional/notification_sample_tests/test_instance.py +++ b/nova/tests/functional/notification_sample_tests/test_instance.py @@ -1262,7 +1262,7 @@ class TestInstanceNotificationSample( 'nova_object.data': {}, 'nova_object.name': 'ImageMetaPropsPayload', 'nova_object.namespace': 'nova', - 'nova_object.version': u'1.1'}, + 'nova_object.version': u'1.2'}, 'image.size': 58145823, 'image.tags': [], 'scheduler_hints': {'_nova_check_type': ['rebuild']}, @@ -1359,7 +1359,7 @@ class TestInstanceNotificationSample( 'nova_object.data': {}, 'nova_object.name': 'ImageMetaPropsPayload', 'nova_object.namespace': 'nova', - 'nova_object.version': u'1.1'}, + 'nova_object.version': u'1.2'}, 'image.size': 58145823, 'image.tags': [], 'scheduler_hints': {'_nova_check_type': ['rebuild']}, diff --git a/nova/tests/unit/api/openstack/fakes.py b/nova/tests/unit/api/openstack/fakes.py index ba75d352672f..4d8a7c6b17a0 100644 --- a/nova/tests/unit/api/openstack/fakes.py +++ b/nova/tests/unit/api/openstack/fakes.py @@ -202,8 +202,9 @@ def stub_out_nw_api(test, cls=None, private=None, publics=None): def validate_networks(self, context, networks, max_count): return max_count - def create_resource_requests(self, context, requested_networks, - pci_requests): + def create_resource_requests( + self, context, requested_networks, + pci_requests=None, affinity_policy=None): return None, [] if cls is None: diff --git a/nova/tests/unit/notifications/objects/test_notification.py b/nova/tests/unit/notifications/objects/test_notification.py index b4c610d01713..4dbcf23b4946 100644 --- a/nova/tests/unit/notifications/objects/test_notification.py +++ b/nova/tests/unit/notifications/objects/test_notification.py @@ -383,7 +383,7 @@ notification_object_data = { 'FlavorNotification': '1.0-a73147b93b520ff0061865849d3dfa56', 'FlavorPayload': '1.4-2e7011b8b4e59167fe8b7a0a81f0d452', 'ImageMetaPayload': '1.0-0e65beeacb3393beed564a57bc2bc989', - 'ImageMetaPropsPayload': '1.1-789c420945f2cae6ac64ca8dffbcb1b8', + 'ImageMetaPropsPayload': '1.2-f237f65e1f14f05a73481dc4192df3ba', 'InstanceActionNotification': '1.0-a73147b93b520ff0061865849d3dfa56', 'InstanceActionPayload': '1.8-4fa3da9cbf0761f1f700ae578f36dc2f', 'InstanceActionRebuildNotification': diff --git a/nova/tests/unit/objects/test_objects.py b/nova/tests/unit/objects/test_objects.py index 68a16d440d5a..cfcfc71f39b6 100644 --- a/nova/tests/unit/objects/test_objects.py +++ b/nova/tests/unit/objects/test_objects.py @@ -1069,7 +1069,7 @@ object_data = { 'HyperVLiveMigrateData': '1.4-e265780e6acfa631476c8170e8d6fce0', 'IDEDeviceBus': '1.0-29d4c9f27ac44197f01b6ac1b7e16502', 'ImageMeta': '1.8-642d1b2eb3e880a367f37d72dd76162d', - 'ImageMetaProps': '1.24-f92fa09d54185499da98f5430524964e', + 'ImageMetaProps': '1.25-66fc973af215eb5701ed4034bb6f0685', 'Instance': '2.7-d187aec68cad2e4d8b8a03a68e4739ce', 'InstanceAction': '1.2-9a5abc87fdd3af46f45731960651efb5', 'InstanceActionEvent': '1.3-c749e1b3589e7117c81cb2aa6ac438d5', diff --git a/nova/tests/unit/pci/test_request.py b/nova/tests/unit/pci/test_request.py index 89b8b1c1042d..fb5eae77c587 100644 --- a/nova/tests/unit/pci/test_request.py +++ b/nova/tests/unit/pci/test_request.py @@ -23,6 +23,7 @@ from nova import context from nova import exception from nova.network import model from nova import objects +from nova.objects import fields from nova.pci import request from nova import test from nova.tests.unit.api.openstack import fakes @@ -189,6 +190,23 @@ class PciRequestTestCase(test.NoDBTestCase): self.assertRaises(exception.PciInvalidAlias, request._get_alias_from_config) + def test_valid_numa_policy(self): + for policy in fields.PCINUMAAffinityPolicy.ALL: + self.flags(alias=[ + """{ + "name": "xxx", + "capability_type": "pci", + "product_id": "1111", + "vendor_id": "8086", + "device_type": "type-PCI", + "numa_policy": "%s" + }""" % policy], + group='pci') + aliases = request._get_alias_from_config() + self.assertIsNotNone(aliases) + self.assertIn("xxx", aliases) + self.assertEqual(policy, aliases["xxx"][0]) + def test_conflicting_device_type(self): """Check behavior when device_type conflicts occur.""" self.flags(alias=[ @@ -268,6 +286,37 @@ class PciRequestTestCase(test.NoDBTestCase): request._translate_alias_to_requests, "QuicAssistX : 3") + def test_alias_2_request_affinity_policy(self): + # _fake_alias1 requests the legacy policy and _fake_alias3 + # has no numa_policy set so it will default to legacy. + self.flags(alias=[_fake_alias1, _fake_alias3], group='pci') + # so to test that the flavor/image policy takes precedence + # set use the preferred policy. + policy = fields.PCINUMAAffinityPolicy.PREFERRED + expect_request = [ + {'count': 3, + 'requester_id': None, + 'spec': [{'vendor_id': '8086', 'product_id': '4443', + 'dev_type': 'type-PCI', + 'capability_type': 'pci'}], + 'alias_name': 'QuicAssist', + 'numa_policy': policy + }, + + {'count': 1, + 'requester_id': None, + 'spec': [{'vendor_id': '8086', 'product_id': '1111', + 'dev_type': "type-PF", + 'capability_type': 'pci'}], + 'alias_name': 'IntelNIC', + 'numa_policy': policy + }, ] + + requests = request._translate_alias_to_requests( + "QuicAssist : 3, IntelNIC: 1", affinity_policy=policy) + self.assertEqual(set([p['count'] for p in requests]), set([1, 3])) + self._verify_result(expect_request, requests) + @mock.patch.object(objects.compute_node.ComputeNode, 'get_by_host_and_nodename') def test_get_instance_pci_request_from_vif_invalid( @@ -410,3 +459,14 @@ class PciRequestTestCase(test.NoDBTestCase): flavor = {} requests = request.get_pci_requests_from_flavor(flavor) self.assertEqual([], requests.requests) + + @mock.patch.object( + request, "_translate_alias_to_requests", return_value=[]) + def test_get_pci_requests_from_flavor_affinity_policy( + self, mock_translate): + self.flags(alias=[_fake_alias1, _fake_alias3], group='pci') + flavor = {'extra_specs': {"pci_passthrough:alias": + "QuicAssist:3, IntelNIC: 1"}} + policy = fields.PCINUMAAffinityPolicy.PREFERRED + request.get_pci_requests_from_flavor(flavor, affinity_policy=policy) + mock_translate.assert_called_with(mock.ANY, affinity_policy=policy) diff --git a/nova/tests/unit/virt/libvirt/fakelibvirt.py b/nova/tests/unit/virt/libvirt/fakelibvirt.py index 0848fc656599..34fc2643c7c2 100644 --- a/nova/tests/unit/virt/libvirt/fakelibvirt.py +++ b/nova/tests/unit/virt/libvirt/fakelibvirt.py @@ -305,6 +305,11 @@ class FakePCIDevice(object): 'iommu_group': iommu_group, 'numa_node': numa_node, } + # -1 is the sentinel set in /sys/bus/pci/devices/*/numa_node + # for no NUMA affinity. When the numa_node is set to -1 on a device + # Libvirt omits the NUMA element so we remove it. + if numa_node == -1: + self.pci_device = self.pci_device.replace("", "") def XMLDesc(self, flags): return self.pci_device diff --git a/nova/tests/unit/virt/test_hardware.py b/nova/tests/unit/virt/test_hardware.py index b08244dcacef..e4963078b776 100644 --- a/nova/tests/unit/virt/test_hardware.py +++ b/nova/tests/unit/virt/test_hardware.py @@ -16,6 +16,7 @@ import collections import copy import mock +import testtools from nova import exception from nova import objects @@ -4184,3 +4185,73 @@ class MemEncryptionRequiredTestCase(test.NoDBTestCase): "hw_mem_encryption property of image %s" % (self.flavor_name, self.image_name) ) + + +class PCINUMAAffinityPolicyTest(test.NoDBTestCase): + + def test_get_pci_numa_policy_flavor(self): + + for policy in fields.PCINUMAAffinityPolicy.ALL: + extra_specs = { + "hw:pci_numa_affinity_policy": policy, + } + image_meta = objects.ImageMeta.from_dict({"properties": {}}) + flavor = objects.Flavor( + vcpus=16, memory_mb=2048, extra_specs=extra_specs) + self.assertEqual( + policy, hw.get_pci_numa_policy_constraint(flavor, image_meta)) + + def test_get_pci_numa_policy_image(self): + for policy in fields.PCINUMAAffinityPolicy.ALL: + props = { + "hw_pci_numa_affinity_policy": policy, + } + image_meta = objects.ImageMeta.from_dict({"properties": props}) + flavor = objects.Flavor( + vcpus=16, memory_mb=2048, extra_specs={}) + self.assertEqual( + policy, hw.get_pci_numa_policy_constraint(flavor, image_meta)) + + def test_get_pci_numa_policy_no_conflict(self): + + for policy in fields.PCINUMAAffinityPolicy.ALL: + extra_specs = { + "hw:pci_numa_affinity_policy": policy, + } + flavor = objects.Flavor( + vcpus=16, memory_mb=2048, extra_specs=extra_specs) + props = { + "hw_pci_numa_affinity_policy": policy, + } + image_meta = objects.ImageMeta.from_dict({"properties": props}) + self.assertEqual( + policy, hw.get_pci_numa_policy_constraint(flavor, image_meta)) + + def test_get_pci_numa_policy_conflict(self): + extra_specs = { + "hw:pci_numa_affinity_policy": + fields.PCINUMAAffinityPolicy.LEGACY, + } + flavor = objects.Flavor( + vcpus=16, memory_mb=2048, extra_specs=extra_specs) + props = { + "hw_pci_numa_affinity_policy": + fields.PCINUMAAffinityPolicy.REQUIRED, + } + image_meta = objects.ImageMeta.from_dict({"properties": props}) + self.assertRaises( + exception.ImagePCINUMAPolicyForbidden, + hw.get_pci_numa_policy_constraint, flavor, image_meta) + + def test_get_pci_numa_policy_invalid(self): + extra_specs = { + "hw:pci_numa_affinity_policy": "fake", + } + flavor = objects.Flavor( + vcpus=16, memory_mb=2048, extra_specs=extra_specs) + image_meta = objects.ImageMeta.from_dict({"properties": {}}) + self.assertRaises( + exception.InvalidPCINUMAAffinity, + hw.get_pci_numa_policy_constraint, flavor, image_meta) + with testtools.ExpectedException(ValueError): + image_meta.properties.hw_pci_numa_affinity_policy = "fake" diff --git a/nova/virt/hardware.py b/nova/virt/hardware.py index bcc0c77839e0..473e650ed285 100644 --- a/nova/virt/hardware.py +++ b/nova/virt/hardware.py @@ -1715,6 +1715,28 @@ def get_emulator_thread_policy_constraint(flavor): return emu_threads_policy +def get_pci_numa_policy_constraint(flavor, image_meta): + """Return pci numa affinity policy or None. + + :param flavor: a flavor object to read extra specs from + :param image_meta: nova.objects.ImageMeta object instance + :raises: nova.exception.ImagePCINUMAPolicyForbidden + :raises: nova.exception.InvalidPCINUMAAffinity + """ + flavor_policy, image_policy = _get_flavor_image_meta( + 'pci_numa_affinity_policy', flavor, image_meta) + + if flavor_policy and image_policy and flavor_policy != image_policy: + raise exception.ImagePCINUMAPolicyForbidden() + + policy = flavor_policy or image_policy + + if policy and policy not in fields.PCINUMAAffinityPolicy.ALL: + raise exception.InvalidPCINUMAAffinity(policy=policy) + + return policy + + # TODO(sahid): Move numa related to hardware/numa.py def numa_get_constraints(flavor, image_meta): """Return topology related to input request. diff --git a/releasenotes/notes/sriov-numa-affinity-policy-b49858452827c727.yaml b/releasenotes/notes/sriov-numa-affinity-policy-b49858452827c727.yaml new file mode 100644 index 000000000000..1288c6786ec6 --- /dev/null +++ b/releasenotes/notes/sriov-numa-affinity-policy-b49858452827c727.yaml @@ -0,0 +1,13 @@ +--- +features: + - | + Added support for instance-level PCI NUMA policies using the + ``hw:pci_numa_affinity_policy`` flavor extra spec and + ``hw_pci_numa_affinity_policy`` image metadata property. + These apply to both PCI passthrough and SR-IOV devices, + unlike host-level PCI NUMA policies configured via the + ``alias`` key of the ``[pci] alias`` config option. + See the `VM Scoped SR-IOV NUMA Affinity`_ spec for more + info. + + .. _`VM Scoped SR-IOV NUMA Affinity` : http://specs.openstack.org/openstack/nova-specs/specs/ussuri/approved/vm-scoped-sriov-numa-affinity.html