From 9d6f9e9cd5dcb1dc9205ace9476dcb3a404ac497 Mon Sep 17 00:00:00 2001 From: Claudiu Belu Date: Mon, 16 Jan 2017 11:52:35 +0200 Subject: [PATCH] Hyper-V PCI Passthrough Discrete Device Assignment is a new feature in Windows Server 2016, offering users the possibility of taking some of the PCI Express devices in their systems and pass them through directly to a guest VM. DocImpact: The compute-pci-passthrough page in the admin-guide will have to be updated to include details regarding PCI passthrough on Hyper-V. Co-Authored-By: Iulia Toader Depends-On: I8e7782d3e1e9f8e92406604f05504a7754ffa3c2 Change-Id: I5a243213ff4241b6f70d21a02c606e8fc96ce6e6 Implements: blueprint hyper-v-pci-passthrough --- nova/tests/unit/virt/hyperv/test_hostops.py | 30 ++++++++++++++- nova/tests/unit/virt/hyperv/test_vmops.py | 37 ++++++++++++++++++- nova/virt/hyperv/hostops.py | 26 +++++++++++++ nova/virt/hyperv/vmops.py | 20 +++++++++- ...er-v-pci-passthrough-babf104d6bc2baa6.yaml | 27 ++++++++++++++ 5 files changed, 136 insertions(+), 4 deletions(-) create mode 100644 releasenotes/notes/hyper-v-pci-passthrough-babf104d6bc2baa6.yaml diff --git a/nova/tests/unit/virt/hyperv/test_hostops.py b/nova/tests/unit/virt/hyperv/test_hostops.py index 9967c8c9079e..378d252594b6 100644 --- a/nova/tests/unit/virt/hyperv/test_hostops.py +++ b/nova/tests/unit/virt/hyperv/test_hostops.py @@ -21,6 +21,7 @@ from oslo_config import cfg from oslo_serialization import jsonutils from oslo_utils import units +from nova.objects import fields as obj_fields from nova.tests.unit.virt.hyperv import test_base from nova.virt.hyperv import constants from nova.virt.hyperv import hostops @@ -151,6 +152,7 @@ class HostOpsTestCase(test_base.HyperVBaseTestCase): mock_NUMATopology.assert_called_once_with( cells=[mock_NUMACell.return_value]) + @mock.patch.object(hostops.HostOps, '_get_pci_passthrough_devices') @mock.patch.object(hostops.HostOps, '_get_host_numa_topology') @mock.patch.object(hostops.HostOps, '_get_remotefx_gpu_info') @mock.patch.object(hostops.HostOps, '_get_cpu_info') @@ -162,7 +164,8 @@ class HostOpsTestCase(test_base.HyperVBaseTestCase): mock_get_storage_info_gb, mock_get_hypervisor_version, mock_get_memory_info, mock_get_cpu_info, - mock_get_gpu_info, mock_get_numa_topology): + mock_get_gpu_info, mock_get_numa_topology, + mock_get_pci_devices): mock_get_storage_info_gb.return_value = (mock.sentinel.LOCAL_GB, mock.sentinel.LOCAL_GB_FREE, mock.sentinel.LOCAL_GB_USED) @@ -174,6 +177,7 @@ class HostOpsTestCase(test_base.HyperVBaseTestCase): mock_get_hypervisor_version.return_value = mock.sentinel.VERSION mock_get_numa_topology.return_value._to_json.return_value = ( mock.sentinel.numa_topology_json) + mock_get_pci_devices.return_value = mock.sentinel.pcis mock_gpu_info = self._get_mock_gpu_info() mock_get_gpu_info.return_value = mock_gpu_info @@ -183,6 +187,7 @@ class HostOpsTestCase(test_base.HyperVBaseTestCase): mock_get_memory_info.assert_called_once_with() mock_get_cpu_info.assert_called_once_with() mock_get_hypervisor_version.assert_called_once_with() + mock_get_pci_devices.assert_called_once_with() expected = {'supported_instances': [("i686", "hyperv", "hvm"), ("x86_64", "hyperv", "hvm")], 'hypervisor_hostname': mock_node(), @@ -199,9 +204,32 @@ class HostOpsTestCase(test_base.HyperVBaseTestCase): 'remotefx_available_video_ram': 2048, 'remotefx_gpu_info': mock.sentinel.FAKE_GPU_INFO, 'remotefx_total_video_ram': 4096, + 'pci_passthrough_devices': mock.sentinel.pcis, } self.assertEqual(expected, response) + @mock.patch.object(hostops.jsonutils, 'dumps') + def test_get_pci_passthrough_devices(self, mock_jsonutils_dumps): + mock_pci_dev = {'vendor_id': 'fake_vendor_id', + 'product_id': 'fake_product_id', + 'dev_id': 'fake_dev_id', + 'address': 'fake_address'} + mock_get_pcis = self._hostops._hostutils.get_pci_passthrough_devices + mock_get_pcis.return_value = [mock_pci_dev] + + expected_label = 'label_%(vendor_id)s_%(product_id)s' % { + 'vendor_id': mock_pci_dev['vendor_id'], + 'product_id': mock_pci_dev['product_id']} + expected_pci_dev = mock_pci_dev.copy() + expected_pci_dev.update(dev_type=obj_fields.PciDeviceType.STANDARD, + label= expected_label, + numa_node=None) + + result = self._hostops._get_pci_passthrough_devices() + + self.assertEqual(mock_jsonutils_dumps.return_value, result) + mock_jsonutils_dumps.assert_called_once_with([expected_pci_dev]) + def _test_host_power_action(self, action): self._hostops._hostutils.host_power_action = mock.Mock() diff --git a/nova/tests/unit/virt/hyperv/test_vmops.py b/nova/tests/unit/virt/hyperv/test_vmops.py index 968aafc1bee6..a2c4827370b5 100644 --- a/nova/tests/unit/virt/hyperv/test_vmops.py +++ b/nova/tests/unit/virt/hyperv/test_vmops.py @@ -528,6 +528,7 @@ class VMOpsTestCase(test_base.HyperVBaseTestCase): self.assertEqual([], events) mock_is_neutron.assert_called_once_with() + @mock.patch.object(vmops.VMOps, '_attach_pci_devices') @mock.patch.object(vmops.VMOps, '_requires_secure_boot') @mock.patch.object(vmops.VMOps, '_requires_certificate') @mock.patch.object(vmops.VMOps, '_get_instance_vnuma_config') @@ -547,9 +548,11 @@ class VMOpsTestCase(test_base.HyperVBaseTestCase): mock_get_vnuma_config, mock_requires_certificate, mock_requires_secure_boot, + mock_attach_pci_devices, enable_instance_metrics, vm_gen=constants.VM_GEN_1, - vnuma_enabled=False): + vnuma_enabled=False, + pci_requests=None): self.flags(dynamic_memory_ratio=2.0, group='hyperv') self.flags(enable_instance_metrics_collection=enable_instance_metrics, group='hyperv') @@ -563,6 +566,11 @@ class VMOpsTestCase(test_base.HyperVBaseTestCase): flavor = flavor_obj.Flavor(**test_flavor.fake_flavor) mock_instance.flavor = flavor + instance_pci_requests = objects.InstancePCIRequests( + requests=pci_requests or [], instance_uuid=mock_instance.uuid) + mock_instance.pci_requests = instance_pci_requests + host_shutdown_action = (os_win_const.HOST_SHUTDOWN_ACTION_SHUTDOWN + if pci_requests else None) if vnuma_enabled: mock_get_vnuma_config.return_value = ( @@ -590,7 +598,8 @@ class VMOpsTestCase(test_base.HyperVBaseTestCase): self._vmops._vmutils.update_vm.assert_called_once_with( mock_instance.name, mock_instance.flavor.memory_mb, mem_per_numa, mock_instance.flavor.vcpus, cpus_per_numa, - CONF.hyperv.limit_cpu_features, dynamic_memory_ratio) + CONF.hyperv.limit_cpu_features, dynamic_memory_ratio, + host_shutdown_action=host_shutdown_action) mock_configure_remotefx.assert_called_once_with(mock_instance, vm_gen) mock_create_scsi_ctrl = self._vmops._vmutils.create_scsi_controller @@ -617,6 +626,7 @@ class VMOpsTestCase(test_base.HyperVBaseTestCase): enable_secure_boot.assert_called_once_with( mock_instance.name, msft_ca_required=mock_requires_certificate.return_value) + mock_attach_pci_devices.assert_called_once_with(mock_instance) def test_create_instance(self): self._test_create_instance(enable_instance_metrics=True) @@ -632,6 +642,29 @@ class VMOpsTestCase(test_base.HyperVBaseTestCase): self._test_create_instance(enable_instance_metrics=False, vnuma_enabled=True) + def test_create_instance_pci_requested(self): + vendor_id = 'fake_vendor_id' + product_id = 'fake_product_id' + spec = {'vendor_id': vendor_id, 'product_id': product_id} + request = objects.InstancePCIRequest(count=1, spec=[spec]) + self._test_create_instance(enable_instance_metrics=False, + pci_requests=[request]) + + def test_attach_pci_devices(self): + mock_instance = fake_instance.fake_instance_obj(self.context) + vendor_id = 'fake_vendor_id' + product_id = 'fake_product_id' + spec = {'vendor_id': vendor_id, 'product_id': product_id} + request = objects.InstancePCIRequest(count=2, spec=[spec]) + instance_pci_requests = objects.InstancePCIRequests( + requests=[request], instance_uuid=mock_instance.uuid) + mock_instance.pci_requests = instance_pci_requests + + self._vmops._attach_pci_devices(mock_instance) + + self._vmops._vmutils.add_pci_device.assert_has_calls( + [mock.call(mock_instance.name, vendor_id, product_id)] * 2) + @mock.patch.object(vmops.hardware, 'numa_get_constraints') def _check_get_instance_vnuma_config_exception(self, mock_get_numa, numa_cells): diff --git a/nova/virt/hyperv/hostops.py b/nova/virt/hyperv/hostops.py index b245ab0c6c95..ca5fe19a3e1c 100644 --- a/nova/virt/hyperv/hostops.py +++ b/nova/virt/hyperv/hostops.py @@ -176,12 +176,38 @@ class HostOps(object): obj_fields.HVType.HYPERV, obj_fields.VMMode.HVM)], 'numa_topology': self._get_host_numa_topology()._to_json(), + 'pci_passthrough_devices': self._get_pci_passthrough_devices(), } gpu_info = self._get_remotefx_gpu_info() dic.update(gpu_info) return dic + def _get_pci_passthrough_devices(self): + """Get host PCI devices information. + + Obtains PCI devices information and returns it as a JSON string. + + :returns: a JSON string containing a list of the assignable PCI + devices information. + """ + + pci_devices = self._hostutils.get_pci_passthrough_devices() + + for pci_dev in pci_devices: + # NOTE(claudiub): These fields are required by the PCI tracker. + dev_label = 'label_%(vendor_id)s_%(product_id)s' % { + 'vendor_id': pci_dev['vendor_id'], + 'product_id': pci_dev['product_id']} + + # TODO(claudiub): Find a way to associate the PCI devices with + # the NUMA nodes they are in. + pci_dev.update(dev_type=obj_fields.PciDeviceType.STANDARD, + label=dev_label, + numa_node=None) + + return jsonutils.dumps(pci_devices) + def host_power_action(self, action): """Reboots, shuts down or powers up the host.""" if action in [constants.HOST_POWER_ACTION_SHUTDOWN, diff --git a/nova/virt/hyperv/vmops.py b/nova/virt/hyperv/vmops.py index cf978c87f803..f1b692c2769c 100644 --- a/nova/virt/hyperv/vmops.py +++ b/nova/virt/hyperv/vmops.py @@ -367,6 +367,13 @@ class VMOps(object): dynamic_memory_ratio = CONF.hyperv.dynamic_memory_ratio vnuma_enabled = False + if instance.pci_requests.requests: + # NOTE(claudiub): if the instance requires PCI devices, its + # host shutdown action MUST be shutdown. + host_shutdown_action = os_win_const.HOST_SHUTDOWN_ACTION_SHUTDOWN + else: + host_shutdown_action = None + self._vmutils.create_vm(instance_name, vnuma_enabled, vm_gen, @@ -379,7 +386,8 @@ class VMOps(object): instance.flavor.vcpus, cpus_per_numa_node, CONF.hyperv.limit_cpu_features, - dynamic_memory_ratio) + dynamic_memory_ratio, + host_shutdown_action=host_shutdown_action) self._configure_remotefx(instance, vm_gen) @@ -415,6 +423,16 @@ class VMOps(object): self._vmutils.enable_secure_boot( instance.name, msft_ca_required=certificate_required) + self._attach_pci_devices(instance) + + def _attach_pci_devices(self, instance): + for pci_request in instance.pci_requests.requests: + spec = pci_request.spec[0] + for counter in range(pci_request.count): + self._vmutils.add_pci_device(instance.name, + spec['vendor_id'], + spec['product_id']) + def _get_instance_vnuma_config(self, instance, image_meta): """Returns the appropriate NUMA configuration for Hyper-V instances, given the desired instance NUMA topology. diff --git a/releasenotes/notes/hyper-v-pci-passthrough-babf104d6bc2baa6.yaml b/releasenotes/notes/hyper-v-pci-passthrough-babf104d6bc2baa6.yaml new file mode 100644 index 000000000000..0a0525e505f5 --- /dev/null +++ b/releasenotes/notes/hyper-v-pci-passthrough-babf104d6bc2baa6.yaml @@ -0,0 +1,27 @@ +--- +features: + - | + The nova Hyper-V driver now supports adding PCI passthrough devices to + Hyper-V instances (discrete device assignment). This feature has been + introduced in Windows / Hyper-V Server 2016 and offers the possibility to + attach some of the host's PCI devices (e.g.: GPU devices) directly to + Hyper-V instances. + In order to benefit from this feature, Hyper-V compute nodes must support + SR-IOV and must have assignable PCI devices. This can easily be checked by + running the following powershell commands:: + + Start-BitsTransfer https://raw.githubusercontent.com/Microsoft/Virtualization-Documentation/master/hyperv-samples/benarm-powershell/DDA/survey-dda.ps1 + .\survey-dda.ps1 + + The script above will print a list of assignable PCI devices available on + the host, and if the host supports SR-IOV. + + If the host supports this feature and it has at least an assignable PCI + device, the host must be configured to allow those PCI devices to be + assigned to VMs. For information on how to do this, follow this guide [1]. + + After the compute nodes have been configured, the nova-api, nova-scheduler, + and the nova-compute services will have to be configured next [2]. + + [1] https://blogs.technet.microsoft.com/heyscriptingguy/2016/07/14/passing-through-devices-to-hyper-v-vms-by-using-discrete-device-assignment/ + [2] http://docs.openstack.org/admin-guide/compute-pci-passthrough.html