From 22fcfcda235a7b3534354cbf3fff9e49ee9d2ee3 Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Thu, 11 Mar 2021 15:11:05 +0000 Subject: [PATCH] tests: Add functional test for vDPA device Add a simple test to validate behavior with vDPA devices. Most of this is simply fleshing out the fixtures we use to fake out vDPA devices and generally tweaking things to make them better. Change-Id: I1423d8a9652751b667463f90c69eae1a054dd776 Signed-off-by: Stephen Finucane Co-authored-by: Sean Mooney --- nova/pci/stats.py | 8 +- nova/tests/functional/integrated_helpers.py | 18 + nova/tests/functional/libvirt/base.py | 12 +- .../libvirt/test_pci_sriov_servers.py | 221 +++++++++++- nova/tests/unit/virt/libvirt/fakelibvirt.py | 323 +++++++++++++----- nova/virt/libvirt/host.py | 3 +- 6 files changed, 483 insertions(+), 102 deletions(-) diff --git a/nova/pci/stats.py b/nova/pci/stats.py index bac61dae7b82..56a3b02a0724 100644 --- a/nova/pci/stats.py +++ b/nova/pci/stats.py @@ -441,7 +441,7 @@ class PciDeviceStats(object): if after_count < before_count: LOG.debug( - 'Dropped %d devices due to mismatched PCI attribute(s)', + 'Dropped %d device(s) due to mismatched PCI attribute(s)', before_count - after_count ) @@ -458,7 +458,7 @@ class PciDeviceStats(object): if after_count < before_count: LOG.debug( - 'Dropped %d devices as they are on the wrong NUMA node(s)', + 'Dropped %d device(s) as they are on the wrong NUMA node(s)', before_count - after_count ) @@ -474,7 +474,7 @@ class PciDeviceStats(object): if after_count < before_count: LOG.debug( - 'Dropped %d devices as they are PFs which we have not ' + 'Dropped %d device(s) as they are PFs which we have not ' 'requested', before_count - after_count ) @@ -491,7 +491,7 @@ class PciDeviceStats(object): if after_count < before_count: LOG.debug( - 'Dropped %d devices as they are VDPA devices which we have ' + 'Dropped %d device(s) as they are VDPA devices which we have ' 'not requested', before_count - after_count ) diff --git a/nova/tests/functional/integrated_helpers.py b/nova/tests/functional/integrated_helpers.py index 833c47a2e304..fbf1a551939f 100644 --- a/nova/tests/functional/integrated_helpers.py +++ b/nova/tests/functional/integrated_helpers.py @@ -413,6 +413,24 @@ class InstanceHelperMixin: fake_notifier.wait_for_versioned_notifications('instance.reboot.end') return self._wait_for_state_change(server, expected_state) + def _attach_interface(self, server, port_uuid): + """attach a neutron port to a server.""" + body = { + "interfaceAttachment": { + "port_id": port_uuid + } + } + attachment = self.api.attach_interface(server['id'], body) + fake_notifier.wait_for_versioned_notifications( + 'instance.interface_attach.end') + return attachment + + def _detach_interface(self, server, port_uuid): + """detach a neutron port form a server.""" + self.api.detach_interface(server['id'], port_uuid) + fake_notifier.wait_for_versioned_notifications( + 'instance.interface_detach.end') + def _rebuild_server(self, server, image_uuid, expected_state='ACTIVE'): """Rebuild a server.""" self.api.post_server_action( diff --git a/nova/tests/functional/libvirt/base.py b/nova/tests/functional/libvirt/base.py index d40995cea894..74e47f2e8fc1 100644 --- a/nova/tests/functional/libvirt/base.py +++ b/nova/tests/functional/libvirt/base.py @@ -85,7 +85,7 @@ class ServersTestBase(integrated_helpers._IntegratedTestBase): return self.start_service('scheduler') def _get_connection( - self, host_info=None, pci_info=None, mdev_info=None, + self, host_info=None, pci_info=None, mdev_info=None, vdpa_info=None, libvirt_version=None, qemu_version=None, hostname=None, ): if not host_info: @@ -107,12 +107,14 @@ class ServersTestBase(integrated_helpers._IntegratedTestBase): host_info=host_info, pci_info=pci_info, mdev_info=mdev_info, + vdpa_info=vdpa_info, hostname=hostname) return fake_connection def start_compute( self, hostname='compute1', host_info=None, pci_info=None, - mdev_info=None, libvirt_version=None, qemu_version=None, + mdev_info=None, vdpa_info=None, libvirt_version=None, + qemu_version=None, ): """Start a compute service. @@ -129,8 +131,8 @@ class ServersTestBase(integrated_helpers._IntegratedTestBase): def _start_compute(hostname, host_info): fake_connection = self._get_connection( - host_info, pci_info, mdev_info, libvirt_version, qemu_version, - hostname, + host_info, pci_info, mdev_info, vdpa_info, libvirt_version, + qemu_version, hostname, ) # This is fun. Firstly we need to do a global'ish mock so we can # actually start the service. @@ -299,8 +301,8 @@ class LibvirtNeutronFixture(nova_fixtures.NeutronFixture): 'subnet_id': subnet_4['id'] } ], - 'binding:vif_type': 'hw_veb', 'binding:vif_details': {'vlan': 42}, + 'binding:vif_type': 'hw_veb', 'binding:vnic_type': 'direct', } diff --git a/nova/tests/functional/libvirt/test_pci_sriov_servers.py b/nova/tests/functional/libvirt/test_pci_sriov_servers.py index 44ff408d0d08..3fdc5c6e20b7 100644 --- a/nova/tests/functional/libvirt/test_pci_sriov_servers.py +++ b/nova/tests/functional/libvirt/test_pci_sriov_servers.py @@ -23,6 +23,7 @@ import mock from oslo_config import cfg from oslo_log import log as logging from oslo_serialization import jsonutils +from oslo_utils.fixture import uuidsentinel as uuids from oslo_utils import units import nova @@ -275,9 +276,7 @@ class SRIOVServersTest(_PCIServersTestBase): self.assertNotIn('binding:profile', port) # create a server using the VF via neutron - flavor_id = self._create_flavor() self._create_server( - flavor_id=flavor_id, networks=[ {'port': base.LibvirtNeutronFixture.network_4_port_1['id']}, ], @@ -548,9 +547,7 @@ class SRIOVServersTest(_PCIServersTestBase): self.neutron.create_port({'port': self.neutron.network_4_port_1}) # create a server using the VF via neutron - flavor_id = self._create_flavor() self._create_server( - flavor_id=flavor_id, networks=[ {'port': base.LibvirtNeutronFixture.network_4_port_1['id']}, ], @@ -672,6 +669,222 @@ class SRIOVAttachDetachTest(_PCIServersTestBase): self.neutron.sriov_pf_port2['id']) +class VDPAServersTest(_PCIServersTestBase): + + # this is needed for os_compute_api:os-migrate-server:migrate policy + ADMIN_API = True + microversion = 'latest' + + # Whitelist both the PF and VF; in reality, you probably wouldn't do this + # but we want to make sure that the PF is correctly taken off the table + # once any VF is used + PCI_PASSTHROUGH_WHITELIST = [jsonutils.dumps(x) for x in ( + { + 'vendor_id': '15b3', + 'product_id': '101d', + 'physical_network': 'physnet4', + }, + { + 'vendor_id': '15b3', + 'product_id': '101e', + 'physical_network': 'physnet4', + }, + )] + # No need for aliases as these test will request SRIOV via neutron + PCI_ALIAS = [] + + NUM_PFS = 1 + NUM_VFS = 4 + + FAKE_LIBVIRT_VERSION = 6_009_000 # 6.9.0 + FAKE_QEMU_VERSION = 5_001_000 # 5.1.0 + + def setUp(self): + super().setUp() + + # The ultimate base class _IntegratedTestBase uses NeutronFixture but + # we need a bit more intelligent neutron for these tests. Applying the + # new fixture here means that we re-stub what the previous neutron + # fixture already stubbed. + self.neutron = self.useFixture(base.LibvirtNeutronFixture(self)) + + def start_compute(self): + vf_ratio = self.NUM_VFS // self.NUM_PFS + + pci_info = fakelibvirt.HostPCIDevicesInfo( + num_pci=0, num_pfs=0, num_vfs=0) + vdpa_info = fakelibvirt.HostVDPADevicesInfo() + + pci_info.add_device( + dev_type='PF', + bus=0x6, + slot=0x0, + function=0, + iommu_group=40, # totally arbitrary number + numa_node=0, + vf_ratio=vf_ratio, + vend_id='15b3', + vend_name='Mellanox Technologies', + prod_id='101d', + prod_name='MT2892 Family [ConnectX-6 Dx]', + driver_name='mlx5_core') + + for idx in range(self.NUM_VFS): + vf = pci_info.add_device( + dev_type='VF', + bus=0x6, + slot=0x0, + function=idx + 1, + iommu_group=idx + 41, # totally arbitrary number + offset + numa_node=0, + vf_ratio=vf_ratio, + parent=(0x6, 0x0, 0), + vend_id='15b3', + vend_name='Mellanox Technologies', + prod_id='101e', + prod_name='ConnectX Family mlx5Gen Virtual Function', + driver_name='mlx5_core') + vdpa_info.add_device(f'vdpa_vdpa{idx}', idx, vf) + + return super().start_compute( + pci_info=pci_info, vdpa_info=vdpa_info, + libvirt_version=self.FAKE_LIBVIRT_VERSION, + qemu_version=self.FAKE_QEMU_VERSION) + + def create_vdpa_port(self): + vdpa_port = { + 'id': uuids.vdpa_port, + 'network_id': self.neutron.network_4['id'], + 'status': 'ACTIVE', + 'mac_address': 'b5:bc:2e:e7:51:ee', + 'fixed_ips': [ + { + 'ip_address': '192.168.4.6', + 'subnet_id': self.neutron.subnet_4['id'] + } + ], + 'binding:vif_details': {}, + 'binding:vif_type': 'ovs', + 'binding:vnic_type': 'vdpa', + } + + # create the port + self.neutron.create_port({'port': vdpa_port}) + return vdpa_port + + def test_create_server(self): + """Create an instance using a neutron-provisioned vDPA VIF.""" + + orig_create = nova.virt.libvirt.guest.Guest.create + + def fake_create(cls, xml, host): + tree = etree.fromstring(xml) + elem = tree.find('./devices/interface/[@type="vdpa"]') + + # compare source device + # the MAC address is derived from the neutron port, while the + # source dev path assumes we attach vDPA devs in order + expected = """ + + + + + """ + actual = etree.tostring(elem, encoding='unicode') + + self.assertXmlEqual(expected, actual) + + return orig_create(xml, host) + + self.stub_out( + 'nova.virt.libvirt.guest.Guest.create', + fake_create, + ) + + hostname = self.start_compute() + num_pci = self.NUM_PFS + self.NUM_VFS + + # both the PF and VF with vDPA capabilities (dev_type=vdpa) should have + # been counted + self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci) + + # create the port + vdpa_port = self.create_vdpa_port() + + # ensure the binding details are currently unset + port = self.neutron.show_port(vdpa_port['id'])['port'] + self.assertNotIn('binding:profile', port) + + # create a server using the vDPA device via neutron + self._create_server(networks=[{'port': vdpa_port['id']}]) + + # ensure there is one less VF available and that the PF is no longer + # usable + self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci - 2) + + # ensure the binding details sent to "neutron" were correct + port = self.neutron.show_port(vdpa_port['id'])['port'] + self.assertIn('binding:profile', port) + self.assertEqual( + { + 'pci_vendor_info': '15b3:101e', + 'pci_slot': '0000:06:00.4', + 'physical_network': 'physnet4', + }, + port['binding:profile'], + ) + + def _test_common(self, op, *args, **kwargs): + self.start_compute() + + # create the port and a server, with the port attached to the server + vdpa_port = self.create_vdpa_port() + server = self._create_server(networks=[{'port': vdpa_port['id']}]) + + # attempt the unsupported action and ensure it fails + ex = self.assertRaises( + client.OpenStackApiException, + op, server, *args, **kwargs) + self.assertIn( + 'not supported for instance with vDPA ports', + ex.response.text) + + def test_attach_interface(self): + self.start_compute() + + # create the port and a server, but don't attach the port to the server + # yet + vdpa_port = self.create_vdpa_port() + server = self._create_server(networks='none') + + # attempt to attach the port to the server + ex = self.assertRaises( + client.OpenStackApiException, + self._attach_interface, server, vdpa_port['id']) + self.assertIn( + 'not supported for instance with vDPA ports', + ex.response.text) + + def test_detach_interface(self): + self._test_common(self._detach_interface, uuids.vdpa_port) + + def test_shelve(self): + self._test_common(self._shelve_server) + + def test_suspend(self): + self._test_common(self._suspend_server) + + def test_evacute(self): + self._test_common(self._evacuate_server) + + def test_resize(self): + flavor_id = self._create_flavor() + self._test_common(self._resize_server, flavor_id) + + def test_cold_migrate(self): + self._test_common(self._migrate_server) + + class PCIServersTest(_PCIServersTestBase): ADMIN_API = True diff --git a/nova/tests/unit/virt/libvirt/fakelibvirt.py b/nova/tests/unit/virt/libvirt/fakelibvirt.py index 718dad8b106a..115b490984d1 100644 --- a/nova/tests/unit/virt/libvirt/fakelibvirt.py +++ b/nova/tests/unit/virt/libvirt/fakelibvirt.py @@ -185,7 +185,7 @@ VIR_CONNECT_LIST_DOMAINS_INACTIVE = 2 # virConnectListAllNodeDevices flags VIR_CONNECT_LIST_NODE_DEVICES_CAP_PCI_DEV = 2 -VIR_CONNECT_LIST_NODE_DEVICES_CAP_NET = 16 +VIR_CONNECT_LIST_NODE_DEVICES_CAP_NET = 1 << 4 VIR_CONNECT_LIST_NODE_DEVICES_CAP_VDPA = 1 << 17 # secret type @@ -241,6 +241,11 @@ os_uname = collections.namedtuple( ) +def _get_libvirt_nodedev_name(bus, slot, function): + """Convert an address to a libvirt device name string.""" + return f'pci_0000_{bus:02x}_{slot:02x}_{function:d}' + + class FakePCIDevice(object): """Generate a fake PCI device. @@ -255,22 +260,22 @@ class FakePCIDevice(object): pci_default_parent = "pci_0000_80_01_0" pci_device_template = textwrap.dedent(""" - pci_0000_81_%(slot)02x_%(function)d - /sys/devices/pci0000:80/0000:80:01.0/0000:81:%(slot)02x.%(function)d + pci_0000_%(bus)02x_%(slot)02x_%(function)d + /sys/devices/pci0000:80/0000:80:01.0/0000:%(bus)02x:%(slot)02x.%(function)d %(parent)s %(driver)s 0 - 129 + %(bus)d %(slot)d %(function)d %(prod_name)s %(vend_name)s %(capability)s -
+
@@ -280,7 +285,7 @@ class FakePCIDevice(object): """.strip()) # noqa cap_templ = "%(addresses)s" - addr_templ = "
" # noqa + addr_templ = "
" # noqa mdevtypes_templ = textwrap.dedent(""" GRID M60-0Bvfio-pci @@ -289,22 +294,35 @@ class FakePCIDevice(object): is_capable_of_mdevs = False - def __init__(self, dev_type, slot, function, iommu_group, numa_node, - vf_ratio=None, multiple_gpu_types=False, parent=None): + def __init__( + self, dev_type, bus, slot, function, iommu_group, numa_node, *, + vf_ratio=None, multiple_gpu_types=False, parent=None, + vend_id=None, vend_name=None, prod_id=None, prod_name=None, + driver_name=None, + ): """Populate pci devices - :param dev_type: (string) Indicates the type of the device (PCI, PF, - VF). + :param dev_type: (str) Indicates the type of the device (PCI, PF, VF, + MDEV_TYPES). + :param bus: (int) Bus number of the device. :param slot: (int) Slot number of the device. :param function: (int) Function number of the device. :param iommu_group: (int) IOMMU group ID. :param numa_node: (int) NUMA node of the device. :param vf_ratio: (int) Ratio of Virtual Functions on Physical. Only applicable if ``dev_type`` is one of: ``PF``, ``VF``. - :param multiple_gpu_types: (bool) Supports different vGPU types + :param multiple_gpu_types: (bool) Supports different vGPU types. + :param parent: (int, int, int) A tuple of bus, slot and function + corresponding to the parent. + :param vend_id: (str) The vendor ID. + :param vend_name: (str) The vendor name. + :param prod_id: (str) The product ID. + :param prod_name: (str) The product name. + :param driver_name: (str) The driver name. """ self.dev_type = dev_type + self.bus = bus self.slot = slot self.function = function self.iommu_group = iommu_group @@ -312,28 +330,49 @@ class FakePCIDevice(object): self.vf_ratio = vf_ratio self.multiple_gpu_types = multiple_gpu_types self.parent = parent + + self.vend_id = vend_id + self.vend_name = vend_name + self.prod_id = prod_id + self.prod_name = prod_name + self.driver_name = driver_name + self.generate_xml() def generate_xml(self, skip_capability=False): - vend_id = PCI_VEND_ID - vend_name = PCI_VEND_NAME + + # initial validation + assert self.dev_type in ('PCI', 'VF', 'PF', 'MDEV_TYPES'), ( + f'got invalid dev_type {self.dev_type}') + + if self.dev_type == 'PCI': + assert not self.vf_ratio, 'vf_ratio does not apply for PCI devices' + + if self.dev_type in ('PF', 'VF'): + assert self.vf_ratio, 'require vf_ratio for PFs and VFs' + + if self.dev_type == 'VF': + assert self.parent, 'require parent for VFs' + assert isinstance(self.parent, tuple), 'parent must be an address' + assert len(self.parent) == 3, 'parent must be an address' + + vend_id = self.vend_id or PCI_VEND_ID + vend_name = self.vend_name or PCI_VEND_NAME capability = '' if self.dev_type == 'PCI': - if self.vf_ratio: - raise ValueError('vf_ratio does not apply for PCI devices') - - prod_id = PCI_PROD_ID - prod_name = PCI_PROD_NAME - driver = PCI_DRIVER_NAME + prod_id = self.prod_id or PCI_PROD_ID + prod_name = self.prod_name or PCI_PROD_NAME + driver = self.driver_name or PCI_DRIVER_NAME elif self.dev_type == 'PF': - prod_id = PF_PROD_ID - prod_name = PF_PROD_NAME - driver = PF_DRIVER_NAME + prod_id = self.prod_id or PF_PROD_ID + prod_name = self.prod_name or PF_PROD_NAME + driver = self.driver_name or PF_DRIVER_NAME if not skip_capability: capability = self.cap_templ % { 'cap_type': PF_CAP_TYPE, 'addresses': '\n'.join([ self.addr_templ % { + 'bus': self.bus, # these are the slot, function values of the child # VFs, we can only assign 8 functions to a slot # (0-7) so bump the slot each time we exceed this @@ -344,13 +383,14 @@ class FakePCIDevice(object): } for x in range(1, self.vf_ratio + 1)]) } elif self.dev_type == 'VF': - prod_id = VF_PROD_ID - prod_name = VF_PROD_NAME - driver = VF_DRIVER_NAME + prod_id = self.prod_id or VF_PROD_ID + prod_name = self.prod_name or VF_PROD_NAME + driver = self.driver_name or VF_DRIVER_NAME if not skip_capability: capability = self.cap_templ % { 'cap_type': VF_CAP_TYPE, 'addresses': self.addr_templ % { + 'bus': self.bus, # this is the slot, function value of the parent PF # if we're e.g. device 8, we'll have a different slot # to our parent so reverse this @@ -360,11 +400,11 @@ class FakePCIDevice(object): } } elif self.dev_type == 'MDEV_TYPES': - prod_id = MDEV_CAPABLE_PROD_ID - prod_name = MDEV_CAPABLE_PROD_NAME - driver = MDEV_CAPABLE_DRIVER_NAME - vend_id = MDEV_CAPABLE_VEND_ID - vend_name = MDEV_CAPABLE_VEND_NAME + prod_id = self.prod_id or MDEV_CAPABLE_PROD_ID + prod_name = self.prod_name or MDEV_CAPABLE_PROD_NAME + driver = self.driver_name or MDEV_CAPABLE_DRIVER_NAME + vend_id = self.vend_id or MDEV_CAPABLE_VEND_ID + vend_name = self.vend_name or MDEV_CAPABLE_VEND_NAME types = [self.mdevtypes_templ % { 'type_id': NVIDIA_11_VGPU_TYPE, 'instances': 16, @@ -380,10 +420,13 @@ class FakePCIDevice(object): 'addresses': '\n'.join(types) } self.is_capable_of_mdevs = True - else: - raise ValueError('Expected one of: PCI, VF, PCI') + + parent = self.pci_default_parent + if self.parent: + parent = _get_libvirt_nodedev_name(*self.parent) self.pci_device = self.pci_device_template % { + 'bus': self.bus, 'slot': self.slot, 'function': self.function, 'vend_id': vend_id, @@ -394,7 +437,7 @@ class FakePCIDevice(object): 'capability': capability, 'iommu_group': self.iommu_group, 'numa_node': self.numa_node, - 'parent': self.parent or self.pci_default_parent + 'parent': parent, } # -1 is the sentinel set in /sys/bus/pci/devices/*/numa_node # for no NUMA affinity. When the numa_node is set to -1 on a device @@ -406,11 +449,12 @@ class FakePCIDevice(object): return self.pci_device +# TODO(stephenfin): Remove all of these HostFooDevicesInfo objects in favour of +# a unified devices object class HostPCIDevicesInfo(object): """Represent a pool of host PCI devices.""" TOTAL_NUMA_NODES = 2 - pci_devname_template = 'pci_0000_81_%(slot)02x_%(function)d' def __init__(self, num_pci=0, num_pfs=2, num_vfs=8, num_mdevcap=0, numa_node=None, multiple_gpu_types=False): @@ -422,7 +466,6 @@ class HostPCIDevicesInfo(object): :param num_vfs: (int) The number of PCI SR-IOV Virtual Functions. :param num_mdevcap: (int) The number of PCI devices capable of creating mediated devices. - :param iommu_group: (int) Initial IOMMU group ID. :param numa_node: (int) NUMA node of the device; if set all of the devices will be assigned to the specified node else they will be split between ``$TOTAL_NUMA_NODES`` nodes. @@ -439,19 +482,16 @@ class HostPCIDevicesInfo(object): if num_pfs and num_vfs % num_pfs: raise ValueError('num_vfs must be a factor of num_pfs') - slot = 0 + bus = 0x81 + slot = 0x0 function = 0 iommu_group = 40 # totally arbitrary number # Generate PCI devs for dev in range(num_pci): - pci_dev_name = self.pci_devname_template % { - 'slot': slot, 'function': function} - - LOG.info('Generating PCI device %r', pci_dev_name) - - self.devices[pci_dev_name] = FakePCIDevice( + self.add_device( dev_type='PCI', + bus=bus, slot=slot, function=function, iommu_group=iommu_group, @@ -462,13 +502,9 @@ class HostPCIDevicesInfo(object): # Generate MDEV capable devs for dev in range(num_mdevcap): - pci_dev_name = self.pci_devname_template % { - 'slot': slot, 'function': function} - - LOG.info('Generating MDEV capable device %r', pci_dev_name) - - self.devices[pci_dev_name] = FakePCIDevice( + self.add_device( dev_type='MDEV_TYPES', + bus=bus, slot=slot, function=function, iommu_group=iommu_group, @@ -485,19 +521,16 @@ class HostPCIDevicesInfo(object): function = 0 numa_node_pf = self._calc_numa_node(dev, numa_node) - pci_dev_name = self.pci_devname_template % { - 'slot': slot, 'function': function} - - LOG.info('Generating PF device %r', pci_dev_name) - - self.devices[pci_dev_name] = FakePCIDevice( + self.add_device( dev_type='PF', + bus=bus, slot=slot, function=function, iommu_group=iommu_group, numa_node=numa_node_pf, vf_ratio=vf_ratio) - pf_dev_name = pci_dev_name + + parent = (bus, slot, function) # Generate VFs for _ in range(vf_ratio): function += 1 @@ -508,22 +541,46 @@ class HostPCIDevicesInfo(object): slot += 1 function = 0 - pci_dev_name = self.pci_devname_template % { - 'slot': slot, 'function': function} - - LOG.info('Generating VF device %r', pci_dev_name) - - self.devices[pci_dev_name] = FakePCIDevice( + self.add_device( dev_type='VF', + bus=bus, slot=slot, function=function, iommu_group=iommu_group, numa_node=numa_node_pf, vf_ratio=vf_ratio, - parent=pf_dev_name) + parent=parent) slot += 1 + def add_device( + self, dev_type, bus, slot, function, iommu_group, numa_node, + vf_ratio=None, multiple_gpu_types=False, parent=None, + vend_id=None, vend_name=None, prod_id=None, prod_name=None, + driver_name=None, + ): + pci_dev_name = _get_libvirt_nodedev_name(bus, slot, function) + + LOG.info('Generating %s device %r', dev_type, pci_dev_name) + + dev = FakePCIDevice( + dev_type=dev_type, + bus=bus, + slot=slot, + function=function, + iommu_group=iommu_group, + numa_node=numa_node, + vf_ratio=vf_ratio, + multiple_gpu_types=multiple_gpu_types, + parent=parent, + vend_id=vend_id, + vend_name=vend_name, + prod_id=prod_id, + prod_name=prod_name, + driver_name=driver_name) + self.devices[pci_dev_name] = dev + return dev + @classmethod def _calc_numa_node(cls, dev, numa_node): return dev % cls.TOTAL_NUMA_NODES if numa_node is None else numa_node @@ -581,6 +638,68 @@ class HostMdevDevicesInfo(object): return dev +class FakeVDPADevice: + + template = textwrap.dedent(""" + + %(name)s + %(path)s + %(parent)s + + vhost_vdpa + + + /dev/vhost-vdpa-%(idx)d + + """.strip()) + + def __init__(self, name, idx, parent): + assert isinstance(parent, FakePCIDevice) + assert parent.dev_type == 'VF' + + self.name = name + self.idx = idx + self.parent = parent + self.generate_xml() + + def generate_xml(self): + pf_pci = self.parent.parent + vf_pci = (self.parent.bus, self.parent.slot, self.parent.function) + pf_addr = '0000:%02x:%02x.%d' % pf_pci + vf_addr = '0000:%02x:%02x.%d' % vf_pci + parent = _get_libvirt_nodedev_name(*vf_pci) + path = f'/sys/devices/pci0000:00/{pf_addr}/{vf_addr}/vdpa{self.idx}' + self.xml = self.template % { + 'name': self.name, + 'idx': self.idx, + 'path': path, + 'parent': parent, + } + + def XMLDesc(self, flags): + return self.xml + + +class HostVDPADevicesInfo: + + def __init__(self): + self.devices = {} + + def get_all_devices(self): + return self.devices.keys() + + def get_device_by_name(self, device_name): + dev = self.devices[device_name] + return dev + + def add_device(self, name, idx, parent): + LOG.info('Generating vDPA device %r', name) + + dev = FakeVDPADevice(name=name, idx=idx, parent=parent) + self.devices[name] = dev + return dev + + class HostInfo(object): def __init__(self, cpu_nodes=1, cpu_sockets=1, cpu_cores=2, cpu_threads=1, @@ -994,6 +1113,8 @@ class Domain(object): pci_bus, pci_slot, pci_function) nic_info['source'] = pci_device + elif nic_info['type'] == 'vdpa': + nic_info['source'] = source.get('dev') nics_info += [nic_info] @@ -1184,24 +1305,31 @@ class Domain(object): nics = '' for nic in self._def['devices']['nics']: - if 'source' in nic and nic['type'] != 'hostdev': - nics += ''' - - - -
- ''' % nic - # this covers for direct nic type - else: - nics += ''' - - -
- -
- ''' % nic # noqa + if 'source' in nic: + if nic['type'] == 'hostdev': + nics += ''' + + +
+ +
+ ''' % nic # noqa: E501 + elif nic['type'] == 'vdpa': + # TODO(stephenfin): In real life, this would actually have + # an '
' element, but that requires information + # about the host that we're not passing through yet + nics += ''' + + + + ''' + else: + nics += ''' + + + +
+ ''' % nic # noqa: E501 hostdevs = '' for hostdev in self._def['devices']['hostdevs']: @@ -1458,9 +1586,11 @@ class Secret(object): class Connection(object): - def __init__(self, uri=None, readonly=False, version=FAKE_LIBVIRT_VERSION, - hv_version=FAKE_QEMU_VERSION, host_info=None, pci_info=None, - mdev_info=None, hostname=None): + def __init__( + self, uri=None, readonly=False, version=FAKE_LIBVIRT_VERSION, + hv_version=FAKE_QEMU_VERSION, hostname=None, + host_info=None, pci_info=None, mdev_info=None, vdpa_info=None, + ): if not uri or uri == '': if allow_default_uri_connection: uri = 'qemu:///session' @@ -1498,6 +1628,7 @@ class Connection(object): num_pfs=0, num_vfs=0) self.mdev_info = mdev_info or HostMdevDevicesInfo(devices={}) + self.vdpa_info = vdpa_info or HostVDPADevicesInfo() self.hostname = hostname or 'compute1' def _add_nodedev(self, nodedev): @@ -1791,6 +1922,9 @@ class Connection(object): if name.startswith('mdev'): return self.mdev_info.get_device_by_name(name) + if name.startswith('vdpa'): + return self.vdpa_info.get_device_by_name(name) + pci_dev = self.pci_info.get_device_by_name(name) if pci_dev: return pci_dev @@ -1810,6 +1944,8 @@ class Connection(object): return self.mdev_info.get_all_devices() if cap == 'mdev_types': return self.pci_info.get_all_mdev_capable_devices() + if cap == 'vdpa': + return self.vdpa_info.get_all_devices() else: raise ValueError('Capability "%s" is not supported' % cap) @@ -1843,11 +1979,22 @@ class Connection(object): return secret def listAllDevices(self, flags): - # Note this is incomplete as we do not filter - # based on the flags however it is enough for our - # current testing. - return [NodeDevice(self, xml=dev.XMLDesc(0)) - for dev in self.pci_info.devices.values()] + devices = [] + if flags & VIR_CONNECT_LIST_NODE_DEVICES_CAP_PCI_DEV: + devices.extend( + NodeDevice(self, xml=dev.XMLDesc(0)) + for dev in self.pci_info.devices.values() + ) + if flags & VIR_CONNECT_LIST_NODE_DEVICES_CAP_NET: + # TODO(stephenfin): Implement fake netdevs so we can test the + # capability reporting + pass + if flags & VIR_CONNECT_LIST_NODE_DEVICES_CAP_VDPA: + devices.extend( + NodeDevice(self, xml=dev.XMLDesc(0)) + for dev in self.vdpa_info.devices.values() + ) + return devices def openAuth(uri, auth, flags=0): diff --git a/nova/virt/libvirt/host.py b/nova/virt/libvirt/host.py index 42446ad6f750..2321501be912 100644 --- a/nova/virt/libvirt/host.py +++ b/nova/virt/libvirt/host.py @@ -1406,7 +1406,8 @@ class Host(object): raise def list_all_devices( - self, flags: int = 0) -> ty.List['libvirt.virNodeDevice']: + self, flags: int = 0, + ) -> ty.List['libvirt.virNodeDevice']: """Lookup devices. :param flags: a bitmask of flags to filter the returned devices.