diff --git a/nova/tests/functional/libvirt/test_pci_sriov_servers.py b/nova/tests/functional/libvirt/test_pci_sriov_servers.py index 813bfb7fd11e..0a782acba07b 100644 --- a/nova/tests/functional/libvirt/test_pci_sriov_servers.py +++ b/nova/tests/functional/libvirt/test_pci_sriov_servers.py @@ -119,7 +119,7 @@ class SRIOVServersTest(_PCIServersTestBase): host_info = fakelibvirt.NUMAHostInfo(cpu_nodes=2, cpu_sockets=1, cpu_cores=2, cpu_threads=2, kB_mem=15740000) - pci_info = fakelibvirt.HostPciSRIOVDevicesInfo() + pci_info = fakelibvirt.HostPCIDevicesInfo() fake_connection = self._get_connection(host_info, pci_info) self.mock_conn.return_value = fake_connection @@ -135,7 +135,7 @@ class SRIOVServersTest(_PCIServersTestBase): host_info = fakelibvirt.NUMAHostInfo(cpu_nodes=2, cpu_sockets=1, cpu_cores=2, cpu_threads=2, kB_mem=15740000) - pci_info = fakelibvirt.HostPciSRIOVDevicesInfo() + pci_info = fakelibvirt.HostPCIDevicesInfo() fake_connection = self._get_connection(host_info, pci_info) self.mock_conn.return_value = fake_connection @@ -151,7 +151,7 @@ class SRIOVServersTest(_PCIServersTestBase): host_info = fakelibvirt.NUMAHostInfo(cpu_nodes=2, cpu_sockets=1, cpu_cores=2, cpu_threads=2, kB_mem=15740000) - pci_info = fakelibvirt.HostPciSRIOVDevicesInfo(num_pfs=1, num_vfs=4) + pci_info = fakelibvirt.HostPCIDevicesInfo(num_pfs=1, num_vfs=4) fake_connection = self._get_connection(host_info, pci_info) self.mock_conn.return_value = fake_connection @@ -172,7 +172,7 @@ class SRIOVServersTest(_PCIServersTestBase): host_info = fakelibvirt.NUMAHostInfo(cpu_nodes=2, cpu_sockets=1, cpu_cores=2, cpu_threads=2, kB_mem=15740000) - pci_info = fakelibvirt.HostPciSRIOVDevicesInfo(num_pfs=1, num_vfs=4) + pci_info = fakelibvirt.HostPCIDevicesInfo(num_pfs=1, num_vfs=4) fake_connection = self._get_connection(host_info, pci_info) self.mock_conn.return_value = fake_connection @@ -199,7 +199,7 @@ class PCIServersTest(_PCIServersTestBase): host_info = fakelibvirt.NUMAHostInfo(cpu_nodes=2, cpu_sockets=1, cpu_cores=2, cpu_threads=2, kB_mem=15740000) - pci_info = fakelibvirt.HostPciSRIOVDevicesInfo(num_pfs=1, numa_node=1) + pci_info = fakelibvirt.HostPCIDevicesInfo(num_pfs=1, numa_node=1) fake_connection = self._get_connection(host_info, pci_info) self.mock_conn.return_value = fake_connection @@ -221,7 +221,7 @@ class PCIServersTest(_PCIServersTestBase): host_info = fakelibvirt.NUMAHostInfo(cpu_nodes=2, cpu_sockets=1, cpu_cores=2, cpu_threads=2, kB_mem=15740000) - pci_info = fakelibvirt.HostPciSRIOVDevicesInfo(num_pfs=1, numa_node=0) + pci_info = fakelibvirt.HostPCIDevicesInfo(num_pfs=1, numa_node=0) fake_connection = self._get_connection(host_info, pci_info) self.mock_conn.return_value = fake_connection @@ -268,7 +268,7 @@ class PCIServersWithNUMAPoliciesTest(_PCIServersTestBase): host_info = fakelibvirt.NUMAHostInfo(cpu_nodes=2, cpu_sockets=1, cpu_cores=2, cpu_threads=2, kB_mem=15740000) - pci_info = fakelibvirt.HostPciSRIOVDevicesInfo(num_pfs=1, numa_node=0) + pci_info = fakelibvirt.HostPCIDevicesInfo(num_pfs=1, numa_node=0) fake_connection = self._get_connection(host_info, pci_info) self.mock_conn.return_value = fake_connection diff --git a/nova/tests/unit/virt/libvirt/fakelibvirt.py b/nova/tests/unit/virt/libvirt/fakelibvirt.py index 8cde0f9af181..5b64f6054488 100644 --- a/nova/tests/unit/virt/libvirt/fakelibvirt.py +++ b/nova/tests/unit/virt/libvirt/fakelibvirt.py @@ -13,10 +13,12 @@ # under the License. import sys +import textwrap import time import fixtures from lxml import etree +from oslo_log import log as logging from oslo_utils.fixture import uuidsentinel as uuids from nova.objects import fields as obj_fields @@ -34,6 +36,9 @@ def _reset(): global allow_default_uri_connection allow_default_uri_connection = True + +LOG = logging.getLogger(__name__) + # virDomainState VIR_DOMAIN_NOSTATE = 0 VIR_DOMAIN_RUNNING = 1 @@ -163,14 +168,22 @@ FAKE_LIBVIRT_VERSION = 3000000 # Libvirt version to match MIN_QEMU_VERSION in driver.py FAKE_QEMU_VERSION = 2008000 -PF_CAP_TYPE = 'virt_functions' -VF_CAP_TYPE = 'phys_function' +PCI_VEND_ID = '8086' +PCI_VEND_NAME = 'Intel Corporation' + +PCI_PROD_ID = '1533' +PCI_PROD_NAME = 'I210 Gigabit Network Connection' +PCI_DRIVER_NAME = 'igb' + +PF_PROD_ID = '1528' PF_PROD_NAME = 'Ethernet Controller 10-Gigabit X540-AT2' -VF_PROD_NAME = 'X540 Ethernet Controller Virtual Function' PF_DRIVER_NAME = 'ixgbe' +PF_CAP_TYPE = 'virt_functions' + +VF_PROD_ID = '1515' +VF_PROD_NAME = 'X540 Ethernet Controller Virtual Function' VF_DRIVER_NAME = 'ixgbevf' -VF_SLOT = '10' -PF_SLOT = '00' +VF_CAP_TYPE = 'phys_function' NVIDIA_11_VGPU_TYPE = 'nvidia-11' PGPU1_PCI_ADDR = 'pci_0000_06_00_0' @@ -178,117 +191,217 @@ PGPU2_PCI_ADDR = 'pci_0000_07_00_0' PGPU3_PCI_ADDR = 'pci_0000_08_00_0' -class FakePciDevice(object): - pci_dev_template = """ - pci_0000_81_%(slot)02x_%(dev)d - /sys/devices/pci0000:80/0000:80:01.0/0000:81:%(slot)02x.%(dev)d - pci_0000_80_01_0 - - %(driver)s - - - 0 - 129 - %(slot)d - %(dev)d - %(prod_name)s - Intel Corporation - - %(functions)s - - -
- - - - - - - -""" +class FakePCIDevice(object): + """Generate a fake PCI device. - def __init__(self, dev_type, vf_ratio, group, dev, product_id, numa_node): + Generate a fake PCI devices corresponding to one of the following + real-world PCI devices. + + - I210 Gigabit Network Connection (8086:1533) + - Ethernet Controller 10-Gigabit X540-AT2 (8086:1528) + - X540 Ethernet Controller Virtual Function (8086:1515) + """ + + pci_device_template = textwrap.dedent(""" + + pci_0000_81_%(slot)02x_%(function)d + /sys/devices/pci0000:80/0000:80:01.0/0000:81:%(slot)02x.%(function)d + pci_0000_80_01_0 + + %(driver)s + + + 0 + 129 + %(slot)d + %(function)d + %(prod_name)s + %(vend_name)s + %(capability)s + +
+ + + + + + + + """.strip()) # noqa + cap_templ = "%(addresses)s" + addr_templ = "
" # noqa + + def __init__(self, dev_type, slot, function, iommu_group, numa_node, + vf_ratio=None): """Populate pci devices - :param dev_type: (string) Indicates the type of the device (PF, VF) - :param vf_ratio: (int) Ratio of Virtual Functions on Physical - :param group: (int) iommu group id - :param dev: (int) function number of the device - :param product_id: (int) Device product ID - :param numa_node: (int) NUMA node of the device + :param dev_type: (string) Indicates the type of the device (PCI, PF, + VF). + :param slot: (int) Slot number of the device. + :param function: (int) Function number of the device. + :param iommu_group: (int) IOMMU group ID. + :param numa_node: (int) NUMA node of the device. + :param vf_ratio: (int) Ratio of Virtual Functions on Physical. Only + applicable if ``dev_type`` is one of: ``PF``, ``VF``. """ - addr_templ = ("
") - self.pci_dev = None - if dev_type == 'PF': - pf_caps = [addr_templ % {'dev': x, 'slot': VF_SLOT} - for x in range(dev * vf_ratio, - (dev + 1) * vf_ratio)] - slot = int(str(PF_SLOT), 16) - self.pci_dev = self.pci_dev_template % {'dev': dev, - 'prod': product_id, 'group_id': group, - 'functions': '\n'.join(pf_caps), 'slot': slot, - 'cap_type': PF_CAP_TYPE, 'prod_name': PF_PROD_NAME, - 'driver': PF_DRIVER_NAME, 'numa_node': numa_node} + if dev_type == 'PCI': + if vf_ratio: + raise ValueError('vf_ratio does not apply for PCI devices') + + prod_id = PCI_PROD_ID + prod_name = PCI_PROD_NAME + driver = PCI_DRIVER_NAME + capability = '' + elif dev_type == 'PF': + prod_id = PF_PROD_ID + prod_name = PF_PROD_NAME + driver = PF_DRIVER_NAME + capability = self.cap_templ % { + 'cap_type': PF_CAP_TYPE, + 'addresses': '\n'.join([ + self.addr_templ % { + # these are the slot, function values of the child VFs + # we can only assign 8 functions to a slot (0-7) so + # bump the slot each time we exceed this + 'slot': slot + (x // 8), + # ...and wrap the function value + 'function': x % 8, + # the offset is because the PF is occupying function 0 + } for x in range(1, vf_ratio + 1)]) + } elif dev_type == 'VF': - vf_caps = [addr_templ % {'dev': int(dev / vf_ratio), - 'slot': PF_SLOT}] - slot = int(str(VF_SLOT), 16) - self.pci_dev = self.pci_dev_template % {'dev': dev, - 'prod': product_id, 'group_id': group, - 'functions': '\n'.join(vf_caps), 'slot': slot, - 'cap_type': VF_CAP_TYPE, 'prod_name': VF_PROD_NAME, - 'driver': VF_DRIVER_NAME, 'numa_node': numa_node} + prod_id = VF_PROD_ID + prod_name = VF_PROD_NAME + driver = VF_DRIVER_NAME + capability = self.cap_templ % { + 'cap_type': VF_CAP_TYPE, + 'addresses': self.addr_templ % { + # this is the slot, function value of the parent PF + # if we're e.g. device 8, we'll have a different slot + # to our parent so reverse this + 'slot': slot - ((vf_ratio + 1) // 8), + # the parent PF is always function 0 + 'function': 0, + } + } + else: + raise ValueError('Expected one of: PCI, VF, PCI') + + self.pci_device = self.pci_device_template % { + 'slot': slot, + 'function': function, + 'vend_id': PCI_VEND_ID, + 'vend_name': PCI_VEND_NAME, + 'prod_id': prod_id, + 'prod_name': prod_name, + 'driver': driver, + 'capability': capability, + 'iommu_group': iommu_group, + 'numa_node': numa_node, + } def XMLDesc(self, flags): - return self.pci_dev + return self.pci_device -class HostPciSRIOVDevicesInfo(object): - """Represent a pool of host SR-IOV devices.""" +class HostPCIDevicesInfo(object): + """Represent a pool of host PCI devices.""" - def __init__(self, vf_product_id=1515, pf_product_id=1528, num_pfs=2, - num_vfs=8, group=47, numa_node=None, total_numa_nodes=2): - """Create a new HostPciSRIOVDevicesInfo object. + TOTAL_NUMA_NODES = 2 + pci_devname_template = 'pci_0000_81_%(slot)02x_%(function)d' - :param vf_product_id: (int) Product ID of the Virtual Functions - :param pf_product_id=1528: (int) Product ID of the Physical Functions - :param num_pfs: (int) The number of the Physical Functions - :param num_vfs: (int) The number of the Virtual Functions - :param group: (int) Initial group id - :param numa_node: (int) NUMA node of the device, if set all of the - device will be created in the provided node - :param total_numa_nodes: (int) total number of NUMA nodes + def __init__(self, num_pci=0, num_pfs=2, num_vfs=8, numa_node=None): + """Create a new HostPCIDevicesInfo object. + + :param num_pci: (int) The number of (non-SR-IOV) PCI devices. + :param num_pfs: (int) The number of PCI SR-IOV Physical Functions. + :param num_vfs: (int) The number of PCI SR-IOV Virtual Functions. + :param iommu_group: (int) Initial IOMMU group ID. + :param numa_node: (int) NUMA node of the device; if set all of the + devices will be assigned to the specified node else they will be + split between ``$TOTAL_NUMA_NODES`` nodes. """ - def _calc_numa_node(dev): - return dev % total_numa_nodes if numa_node is None else numa_node - self.devices = {} + + if not (num_vfs or num_pfs): + return + if num_vfs and not num_pfs: raise ValueError('Cannot create VFs without PFs') + if num_vfs % num_pfs: + raise ValueError('num_vfs must be a factor of num_pfs') + + slot = 0 + function = 0 + iommu_group = 40 # totally arbitrary number + + # Generate PCI devs + for dev in range(num_pci): + pci_dev_name = self.pci_devname_template % { + 'slot': slot, 'function': function} + + LOG.info('Generating PCI device %r', pci_dev_name) + + self.devices[pci_dev_name] = FakePCIDevice( + dev_type='PCI', + slot=slot, + function=function, + iommu_group=iommu_group, + numa_node=self._calc_numa_node(dev, numa_node)) + + slot += 1 + iommu_group += 1 + vf_ratio = num_vfs // num_pfs if num_pfs else 0 # Generate PFs for dev in range(num_pfs): - dev_group = group + dev + 1 - pci_dev_name = 'pci_0000_81_%(slot)s_%(dev)d' % {'slot': PF_SLOT, - 'dev': dev} - self.devices[pci_dev_name] = FakePciDevice('PF', vf_ratio, - dev_group, dev, - pf_product_id, - _calc_numa_node(dev)) + function = 0 + numa_node_pf = self._calc_numa_node(dev, numa_node) - # Generate VFs - for dev in range(num_vfs): - dev_group = group + dev + 1 - pci_dev_name = 'pci_0000_81_%(slot)s_%(dev)d' % {'slot': VF_SLOT, - 'dev': dev} - self.devices[pci_dev_name] = FakePciDevice('VF', vf_ratio, - dev_group, dev, - vf_product_id, - _calc_numa_node(dev)) + pci_dev_name = self.pci_devname_template % { + 'slot': slot, 'function': function} + + LOG.info('Generating PF device %r', pci_dev_name) + + self.devices[pci_dev_name] = FakePCIDevice( + dev_type='PF', + slot=slot, + function=function, + iommu_group=iommu_group, + numa_node=numa_node_pf, + vf_ratio=vf_ratio) + + # Generate VFs + for _ in range(vf_ratio): + function += 1 + iommu_group += 1 + + if function % 8 == 0: + # functions must be 0-7 + slot += 1 + function = 0 + + pci_dev_name = self.pci_devname_template % { + 'slot': slot, 'function': function} + + LOG.info('Generating VF device %r', pci_dev_name) + + self.devices[pci_dev_name] = FakePCIDevice( + dev_type='VF', + slot=slot, + function=function, + iommu_group=iommu_group, + numa_node=numa_node_pf, + vf_ratio=vf_ratio) + + slot += 1 + + @classmethod + def _calc_numa_node(cls, dev, numa_node): + return dev % cls.TOTAL_NUMA_NODES if numa_node is None else numa_node def get_all_devices(self): return self.devices.keys() @@ -1111,8 +1224,9 @@ class Connection(object): self.fakeLibVersion = version self.fakeVersion = hv_version self.host_info = host_info or HostInfo() - self.pci_info = pci_info or HostPciSRIOVDevicesInfo(num_pfs=0, - num_vfs=0) + self.pci_info = pci_info or HostPCIDevicesInfo(num_pci=0, + num_pfs=0, + num_vfs=0) self.mdev_info = mdev_info or [] def _add_filter(self, nwfilter): diff --git a/nova/tests/unit/virt/libvirt/test_fakelibvirt.py b/nova/tests/unit/virt/libvirt/test_fakelibvirt.py index 5bc603922204..4914dced1027 100644 --- a/nova/tests/unit/virt/libvirt/test_fakelibvirt.py +++ b/nova/tests/unit/virt/libvirt/test_fakelibvirt.py @@ -18,6 +18,7 @@ import six from nova.objects import fields as obj_fields from nova import test +from nova.tests.unit import matchers import nova.tests.unit.virt.libvirt.fakelibvirt as libvirt from nova.virt.libvirt import config as vconfig @@ -438,10 +439,10 @@ class FakeLibvirtTests(test.NoDBTestCase): Ethernet Controller 10-Gigabit X540-AT2 Intel Corporation -
+
- -
+ +
@@ -451,8 +452,8 @@ class FakeLibvirtTests(test.NoDBTestCase): """ vf_xml = """ - pci_0000_81_10_0 - /sys/devices/pci0000:80/0000:80:01.0/0000:81:10.0 + pci_0000_81_00_1 + /sys/devices/pci0000:80/0000:80:01.0/0000:81:00.1 pci_0000_80_01_0 ixgbevf @@ -460,15 +461,15 @@ class FakeLibvirtTests(test.NoDBTestCase): 0 129 - 16 - 0 + 0 + 1 X540 Ethernet Controller Virtual Function Intel Corporation -
+
- -
+ +
@@ -479,16 +480,16 @@ class FakeLibvirtTests(test.NoDBTestCase): """ # create fake pci devices - pci_info = libvirt.HostPciSRIOVDevicesInfo(num_pfs=1, num_vfs=1) + pci_info = libvirt.HostPCIDevicesInfo(num_pfs=1, num_vfs=1) # generate xml for the created pci devices gen_pf = pci_info.get_device_by_name('pci_0000_81_00_0') - gen_vf = pci_info.get_device_by_name('pci_0000_81_10_0') + gen_vf = pci_info.get_device_by_name('pci_0000_81_00_1') - self.assertEqual(gen_pf.XMLDesc(0), pf_xml) - self.assertEqual(gen_vf.XMLDesc(0), vf_xml) + self.assertThat(gen_pf.XMLDesc(0), matchers.XMLMatches(pf_xml)) + self.assertThat(gen_vf.XMLDesc(0), matchers.XMLMatches(vf_xml)) # parse the generated xml with a libvirt config class and compare # device address _cmp_pci_dev_addr(pf_xml, '0000:81:00.0') - _cmp_pci_dev_addr(vf_xml, '0000:81:10.0') + _cmp_pci_dev_addr(vf_xml, '0000:81:00.1')