Browse Source

Merge "tests: Add functional test for vDPA device"

changes/28/782028/1
Zuul 5 months ago
committed by Gerrit Code Review
parent
commit
70e7aff46e
  1. 8
      nova/pci/stats.py
  2. 18
      nova/tests/functional/integrated_helpers.py
  3. 12
      nova/tests/functional/libvirt/base.py
  4. 221
      nova/tests/functional/libvirt/test_pci_sriov_servers.py
  5. 323
      nova/tests/unit/virt/libvirt/fakelibvirt.py
  6. 3
      nova/virt/libvirt/host.py

8
nova/pci/stats.py

@ -441,7 +441,7 @@ class PciDeviceStats(object):
if after_count < before_count:
LOG.debug(
'Dropped %d devices due to mismatched PCI attribute(s)',
'Dropped %d device(s) due to mismatched PCI attribute(s)',
before_count - after_count
)
@ -458,7 +458,7 @@ class PciDeviceStats(object):
if after_count < before_count:
LOG.debug(
'Dropped %d devices as they are on the wrong NUMA node(s)',
'Dropped %d device(s) as they are on the wrong NUMA node(s)',
before_count - after_count
)
@ -474,7 +474,7 @@ class PciDeviceStats(object):
if after_count < before_count:
LOG.debug(
'Dropped %d devices as they are PFs which we have not '
'Dropped %d device(s) as they are PFs which we have not '
'requested',
before_count - after_count
)
@ -491,7 +491,7 @@ class PciDeviceStats(object):
if after_count < before_count:
LOG.debug(
'Dropped %d devices as they are VDPA devices which we have '
'Dropped %d device(s) as they are VDPA devices which we have '
'not requested',
before_count - after_count
)

18
nova/tests/functional/integrated_helpers.py

@ -413,6 +413,24 @@ class InstanceHelperMixin:
fake_notifier.wait_for_versioned_notifications('instance.reboot.end')
return self._wait_for_state_change(server, expected_state)
def _attach_interface(self, server, port_uuid):
"""attach a neutron port to a server."""
body = {
"interfaceAttachment": {
"port_id": port_uuid
}
}
attachment = self.api.attach_interface(server['id'], body)
fake_notifier.wait_for_versioned_notifications(
'instance.interface_attach.end')
return attachment
def _detach_interface(self, server, port_uuid):
"""detach a neutron port form a server."""
self.api.detach_interface(server['id'], port_uuid)
fake_notifier.wait_for_versioned_notifications(
'instance.interface_detach.end')
def _rebuild_server(self, server, image_uuid, expected_state='ACTIVE'):
"""Rebuild a server."""
self.api.post_server_action(

12
nova/tests/functional/libvirt/base.py

@ -85,7 +85,7 @@ class ServersTestBase(integrated_helpers._IntegratedTestBase):
return self.start_service('scheduler')
def _get_connection(
self, host_info=None, pci_info=None, mdev_info=None,
self, host_info=None, pci_info=None, mdev_info=None, vdpa_info=None,
libvirt_version=None, qemu_version=None, hostname=None,
):
if not host_info:
@ -107,12 +107,14 @@ class ServersTestBase(integrated_helpers._IntegratedTestBase):
host_info=host_info,
pci_info=pci_info,
mdev_info=mdev_info,
vdpa_info=vdpa_info,
hostname=hostname)
return fake_connection
def start_compute(
self, hostname='compute1', host_info=None, pci_info=None,
mdev_info=None, libvirt_version=None, qemu_version=None,
mdev_info=None, vdpa_info=None, libvirt_version=None,
qemu_version=None,
):
"""Start a compute service.
@ -129,8 +131,8 @@ class ServersTestBase(integrated_helpers._IntegratedTestBase):
def _start_compute(hostname, host_info):
fake_connection = self._get_connection(
host_info, pci_info, mdev_info, libvirt_version, qemu_version,
hostname,
host_info, pci_info, mdev_info, vdpa_info, libvirt_version,
qemu_version, hostname,
)
# This is fun. Firstly we need to do a global'ish mock so we can
# actually start the service.
@ -299,8 +301,8 @@ class LibvirtNeutronFixture(nova_fixtures.NeutronFixture):
'subnet_id': subnet_4['id']
}
],
'binding:vif_type': 'hw_veb',
'binding:vif_details': {'vlan': 42},
'binding:vif_type': 'hw_veb',
'binding:vnic_type': 'direct',
}

221
nova/tests/functional/libvirt/test_pci_sriov_servers.py

@ -23,6 +23,7 @@ import mock
from oslo_config import cfg
from oslo_log import log as logging
from oslo_serialization import jsonutils
from oslo_utils.fixture import uuidsentinel as uuids
from oslo_utils import units
import nova
@ -275,9 +276,7 @@ class SRIOVServersTest(_PCIServersTestBase):
self.assertNotIn('binding:profile', port)
# create a server using the VF via neutron
flavor_id = self._create_flavor()
self._create_server(
flavor_id=flavor_id,
networks=[
{'port': base.LibvirtNeutronFixture.network_4_port_1['id']},
],
@ -548,9 +547,7 @@ class SRIOVServersTest(_PCIServersTestBase):
self.neutron.create_port({'port': self.neutron.network_4_port_1})
# create a server using the VF via neutron
flavor_id = self._create_flavor()
self._create_server(
flavor_id=flavor_id,
networks=[
{'port': base.LibvirtNeutronFixture.network_4_port_1['id']},
],
@ -672,6 +669,222 @@ class SRIOVAttachDetachTest(_PCIServersTestBase):
self.neutron.sriov_pf_port2['id'])
class VDPAServersTest(_PCIServersTestBase):
# this is needed for os_compute_api:os-migrate-server:migrate policy
ADMIN_API = True
microversion = 'latest'
# Whitelist both the PF and VF; in reality, you probably wouldn't do this
# but we want to make sure that the PF is correctly taken off the table
# once any VF is used
PCI_PASSTHROUGH_WHITELIST = [jsonutils.dumps(x) for x in (
{
'vendor_id': '15b3',
'product_id': '101d',
'physical_network': 'physnet4',
},
{
'vendor_id': '15b3',
'product_id': '101e',
'physical_network': 'physnet4',
},
)]
# No need for aliases as these test will request SRIOV via neutron
PCI_ALIAS = []
NUM_PFS = 1
NUM_VFS = 4
FAKE_LIBVIRT_VERSION = 6_009_000 # 6.9.0
FAKE_QEMU_VERSION = 5_001_000 # 5.1.0
def setUp(self):
super().setUp()
# The ultimate base class _IntegratedTestBase uses NeutronFixture but
# we need a bit more intelligent neutron for these tests. Applying the
# new fixture here means that we re-stub what the previous neutron
# fixture already stubbed.
self.neutron = self.useFixture(base.LibvirtNeutronFixture(self))
def start_compute(self):
vf_ratio = self.NUM_VFS // self.NUM_PFS
pci_info = fakelibvirt.HostPCIDevicesInfo(
num_pci=0, num_pfs=0, num_vfs=0)
vdpa_info = fakelibvirt.HostVDPADevicesInfo()
pci_info.add_device(
dev_type='PF',
bus=0x6,
slot=0x0,
function=0,
iommu_group=40, # totally arbitrary number
numa_node=0,
vf_ratio=vf_ratio,
vend_id='15b3',
vend_name='Mellanox Technologies',
prod_id='101d',
prod_name='MT2892 Family [ConnectX-6 Dx]',
driver_name='mlx5_core')
for idx in range(self.NUM_VFS):
vf = pci_info.add_device(
dev_type='VF',
bus=0x6,
slot=0x0,
function=idx + 1,
iommu_group=idx + 41, # totally arbitrary number + offset
numa_node=0,
vf_ratio=vf_ratio,
parent=(0x6, 0x0, 0),
vend_id='15b3',
vend_name='Mellanox Technologies',
prod_id='101e',
prod_name='ConnectX Family mlx5Gen Virtual Function',
driver_name='mlx5_core')
vdpa_info.add_device(f'vdpa_vdpa{idx}', idx, vf)
return super().start_compute(
pci_info=pci_info, vdpa_info=vdpa_info,
libvirt_version=self.FAKE_LIBVIRT_VERSION,
qemu_version=self.FAKE_QEMU_VERSION)
def create_vdpa_port(self):
vdpa_port = {
'id': uuids.vdpa_port,
'network_id': self.neutron.network_4['id'],
'status': 'ACTIVE',
'mac_address': 'b5:bc:2e:e7:51:ee',
'fixed_ips': [
{
'ip_address': '192.168.4.6',
'subnet_id': self.neutron.subnet_4['id']
}
],
'binding:vif_details': {},
'binding:vif_type': 'ovs',
'binding:vnic_type': 'vdpa',
}
# create the port
self.neutron.create_port({'port': vdpa_port})
return vdpa_port
def test_create_server(self):
"""Create an instance using a neutron-provisioned vDPA VIF."""
orig_create = nova.virt.libvirt.guest.Guest.create
def fake_create(cls, xml, host):
tree = etree.fromstring(xml)
elem = tree.find('./devices/interface/[@type="vdpa"]')
# compare source device
# the MAC address is derived from the neutron port, while the
# source dev path assumes we attach vDPA devs in order
expected = """
<interface type="vdpa">
<mac address="b5:bc:2e:e7:51:ee"/>
<model type="virtio"/>
<source dev="/dev/vhost-vdpa-3"/>
</interface>"""
actual = etree.tostring(elem, encoding='unicode')
self.assertXmlEqual(expected, actual)
return orig_create(xml, host)
self.stub_out(
'nova.virt.libvirt.guest.Guest.create',
fake_create,
)
hostname = self.start_compute()
num_pci = self.NUM_PFS + self.NUM_VFS
# both the PF and VF with vDPA capabilities (dev_type=vdpa) should have
# been counted
self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci)
# create the port
vdpa_port = self.create_vdpa_port()
# ensure the binding details are currently unset
port = self.neutron.show_port(vdpa_port['id'])['port']
self.assertNotIn('binding:profile', port)
# create a server using the vDPA device via neutron
self._create_server(networks=[{'port': vdpa_port['id']}])
# ensure there is one less VF available and that the PF is no longer
# usable
self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci - 2)
# ensure the binding details sent to "neutron" were correct
port = self.neutron.show_port(vdpa_port['id'])['port']
self.assertIn('binding:profile', port)
self.assertEqual(
{
'pci_vendor_info': '15b3:101e',
'pci_slot': '0000:06:00.4',
'physical_network': 'physnet4',
},
port['binding:profile'],
)
def _test_common(self, op, *args, **kwargs):
self.start_compute()
# create the port and a server, with the port attached to the server
vdpa_port = self.create_vdpa_port()
server = self._create_server(networks=[{'port': vdpa_port['id']}])
# attempt the unsupported action and ensure it fails
ex = self.assertRaises(
client.OpenStackApiException,
op, server, *args, **kwargs)
self.assertIn(
'not supported for instance with vDPA ports',
ex.response.text)
def test_attach_interface(self):
self.start_compute()
# create the port and a server, but don't attach the port to the server
# yet
vdpa_port = self.create_vdpa_port()
server = self._create_server(networks='none')
# attempt to attach the port to the server
ex = self.assertRaises(
client.OpenStackApiException,
self._attach_interface, server, vdpa_port['id'])
self.assertIn(
'not supported for instance with vDPA ports',
ex.response.text)
def test_detach_interface(self):
self._test_common(self._detach_interface, uuids.vdpa_port)
def test_shelve(self):
self._test_common(self._shelve_server)
def test_suspend(self):
self._test_common(self._suspend_server)
def test_evacute(self):
self._test_common(self._evacuate_server)
def test_resize(self):
flavor_id = self._create_flavor()
self._test_common(self._resize_server, flavor_id)
def test_cold_migrate(self):
self._test_common(self._migrate_server)
class PCIServersTest(_PCIServersTestBase):
ADMIN_API = True

323
nova/tests/unit/virt/libvirt/fakelibvirt.py

@ -185,7 +185,7 @@ VIR_CONNECT_LIST_DOMAINS_INACTIVE = 2
# virConnectListAllNodeDevices flags
VIR_CONNECT_LIST_NODE_DEVICES_CAP_PCI_DEV = 2
VIR_CONNECT_LIST_NODE_DEVICES_CAP_NET = 16
VIR_CONNECT_LIST_NODE_DEVICES_CAP_NET = 1 << 4
VIR_CONNECT_LIST_NODE_DEVICES_CAP_VDPA = 1 << 17
# secret type
@ -241,6 +241,11 @@ os_uname = collections.namedtuple(
)
def _get_libvirt_nodedev_name(bus, slot, function):
"""Convert an address to a libvirt device name string."""
return f'pci_0000_{bus:02x}_{slot:02x}_{function:d}'
class FakePCIDevice(object):
"""Generate a fake PCI device.
@ -255,22 +260,22 @@ class FakePCIDevice(object):
pci_default_parent = "pci_0000_80_01_0"
pci_device_template = textwrap.dedent("""
<device>
<name>pci_0000_81_%(slot)02x_%(function)d</name>
<path>/sys/devices/pci0000:80/0000:80:01.0/0000:81:%(slot)02x.%(function)d</path>
<name>pci_0000_%(bus)02x_%(slot)02x_%(function)d</name>
<path>/sys/devices/pci0000:80/0000:80:01.0/0000:%(bus)02x:%(slot)02x.%(function)d</path>
<parent>%(parent)s</parent>
<driver>
<name>%(driver)s</name>
</driver>
<capability type='pci'>
<domain>0</domain>
<bus>129</bus>
<bus>%(bus)d</bus>
<slot>%(slot)d</slot>
<function>%(function)d</function>
<product id='0x%(prod_id)s'>%(prod_name)s</product>
<vendor id='0x%(vend_id)s'>%(vend_name)s</vendor>
%(capability)s
<iommuGroup number='%(iommu_group)d'>
<address domain='0x0000' bus='0x81' slot='%(slot)#02x' function='0x%(function)d'/>
<address domain='0x0000' bus='%(bus)#02x' slot='%(slot)#02x' function='0x%(function)d'/>
</iommuGroup>
<numa node='%(numa_node)s'/>
<pci-express>
@ -280,7 +285,7 @@ class FakePCIDevice(object):
</capability>
</device>""".strip()) # noqa
cap_templ = "<capability type='%(cap_type)s'>%(addresses)s</capability>"
addr_templ = "<address domain='0x0000' bus='0x81' slot='%(slot)#02x' function='%(function)#02x'/>" # noqa
addr_templ = "<address domain='0x0000' bus='%(bus)#02x' slot='%(slot)#02x' function='%(function)#02x'/>" # noqa
mdevtypes_templ = textwrap.dedent("""
<type id='%(type_id)s'>
<name>GRID M60-0B</name><deviceAPI>vfio-pci</deviceAPI>
@ -289,22 +294,35 @@ class FakePCIDevice(object):
is_capable_of_mdevs = False
def __init__(self, dev_type, slot, function, iommu_group, numa_node,
vf_ratio=None, multiple_gpu_types=False, parent=None):
def __init__(
self, dev_type, bus, slot, function, iommu_group, numa_node, *,
vf_ratio=None, multiple_gpu_types=False, parent=None,
vend_id=None, vend_name=None, prod_id=None, prod_name=None,
driver_name=None,
):
"""Populate pci devices
:param dev_type: (string) Indicates the type of the device (PCI, PF,
VF).
:param dev_type: (str) Indicates the type of the device (PCI, PF, VF,
MDEV_TYPES).
:param bus: (int) Bus number of the device.
:param slot: (int) Slot number of the device.
:param function: (int) Function number of the device.
:param iommu_group: (int) IOMMU group ID.
:param numa_node: (int) NUMA node of the device.
:param vf_ratio: (int) Ratio of Virtual Functions on Physical. Only
applicable if ``dev_type`` is one of: ``PF``, ``VF``.
:param multiple_gpu_types: (bool) Supports different vGPU types
:param multiple_gpu_types: (bool) Supports different vGPU types.
:param parent: (int, int, int) A tuple of bus, slot and function
corresponding to the parent.
:param vend_id: (str) The vendor ID.
:param vend_name: (str) The vendor name.
:param prod_id: (str) The product ID.
:param prod_name: (str) The product name.
:param driver_name: (str) The driver name.
"""
self.dev_type = dev_type
self.bus = bus
self.slot = slot
self.function = function
self.iommu_group = iommu_group
@ -312,28 +330,49 @@ class FakePCIDevice(object):
self.vf_ratio = vf_ratio
self.multiple_gpu_types = multiple_gpu_types
self.parent = parent
self.vend_id = vend_id
self.vend_name = vend_name
self.prod_id = prod_id
self.prod_name = prod_name
self.driver_name = driver_name
self.generate_xml()
def generate_xml(self, skip_capability=False):
vend_id = PCI_VEND_ID
vend_name = PCI_VEND_NAME
capability = ''
# initial validation
assert self.dev_type in ('PCI', 'VF', 'PF', 'MDEV_TYPES'), (
f'got invalid dev_type {self.dev_type}')
if self.dev_type == 'PCI':
if self.vf_ratio:
raise ValueError('vf_ratio does not apply for PCI devices')
assert not self.vf_ratio, 'vf_ratio does not apply for PCI devices'
prod_id = PCI_PROD_ID
prod_name = PCI_PROD_NAME
driver = PCI_DRIVER_NAME
if self.dev_type in ('PF', 'VF'):
assert self.vf_ratio, 'require vf_ratio for PFs and VFs'
if self.dev_type == 'VF':
assert self.parent, 'require parent for VFs'
assert isinstance(self.parent, tuple), 'parent must be an address'
assert len(self.parent) == 3, 'parent must be an address'
vend_id = self.vend_id or PCI_VEND_ID
vend_name = self.vend_name or PCI_VEND_NAME
capability = ''
if self.dev_type == 'PCI':
prod_id = self.prod_id or PCI_PROD_ID
prod_name = self.prod_name or PCI_PROD_NAME
driver = self.driver_name or PCI_DRIVER_NAME
elif self.dev_type == 'PF':
prod_id = PF_PROD_ID
prod_name = PF_PROD_NAME
driver = PF_DRIVER_NAME
prod_id = self.prod_id or PF_PROD_ID
prod_name = self.prod_name or PF_PROD_NAME
driver = self.driver_name or PF_DRIVER_NAME
if not skip_capability:
capability = self.cap_templ % {
'cap_type': PF_CAP_TYPE,
'addresses': '\n'.join([
self.addr_templ % {
'bus': self.bus,
# these are the slot, function values of the child
# VFs, we can only assign 8 functions to a slot
# (0-7) so bump the slot each time we exceed this
@ -344,13 +383,14 @@ class FakePCIDevice(object):
} for x in range(1, self.vf_ratio + 1)])
}
elif self.dev_type == 'VF':
prod_id = VF_PROD_ID
prod_name = VF_PROD_NAME
driver = VF_DRIVER_NAME
prod_id = self.prod_id or VF_PROD_ID
prod_name = self.prod_name or VF_PROD_NAME
driver = self.driver_name or VF_DRIVER_NAME
if not skip_capability:
capability = self.cap_templ % {
'cap_type': VF_CAP_TYPE,
'addresses': self.addr_templ % {
'bus': self.bus,
# this is the slot, function value of the parent PF
# if we're e.g. device 8, we'll have a different slot
# to our parent so reverse this
@ -360,11 +400,11 @@ class FakePCIDevice(object):
}
}
elif self.dev_type == 'MDEV_TYPES':
prod_id = MDEV_CAPABLE_PROD_ID
prod_name = MDEV_CAPABLE_PROD_NAME
driver = MDEV_CAPABLE_DRIVER_NAME
vend_id = MDEV_CAPABLE_VEND_ID
vend_name = MDEV_CAPABLE_VEND_NAME
prod_id = self.prod_id or MDEV_CAPABLE_PROD_ID
prod_name = self.prod_name or MDEV_CAPABLE_PROD_NAME
driver = self.driver_name or MDEV_CAPABLE_DRIVER_NAME
vend_id = self.vend_id or MDEV_CAPABLE_VEND_ID
vend_name = self.vend_name or MDEV_CAPABLE_VEND_NAME
types = [self.mdevtypes_templ % {
'type_id': NVIDIA_11_VGPU_TYPE,
'instances': 16,
@ -380,10 +420,13 @@ class FakePCIDevice(object):
'addresses': '\n'.join(types)
}
self.is_capable_of_mdevs = True
else:
raise ValueError('Expected one of: PCI, VF, PCI')
parent = self.pci_default_parent
if self.parent:
parent = _get_libvirt_nodedev_name(*self.parent)
self.pci_device = self.pci_device_template % {
'bus': self.bus,
'slot': self.slot,
'function': self.function,
'vend_id': vend_id,
@ -394,7 +437,7 @@ class FakePCIDevice(object):
'capability': capability,
'iommu_group': self.iommu_group,
'numa_node': self.numa_node,
'parent': self.parent or self.pci_default_parent
'parent': parent,
}
# -1 is the sentinel set in /sys/bus/pci/devices/*/numa_node
# for no NUMA affinity. When the numa_node is set to -1 on a device
@ -406,11 +449,12 @@ class FakePCIDevice(object):
return self.pci_device
# TODO(stephenfin): Remove all of these HostFooDevicesInfo objects in favour of
# a unified devices object
class HostPCIDevicesInfo(object):
"""Represent a pool of host PCI devices."""
TOTAL_NUMA_NODES = 2
pci_devname_template = 'pci_0000_81_%(slot)02x_%(function)d'
def __init__(self, num_pci=0, num_pfs=2, num_vfs=8, num_mdevcap=0,
numa_node=None, multiple_gpu_types=False):
@ -422,7 +466,6 @@ class HostPCIDevicesInfo(object):
:param num_vfs: (int) The number of PCI SR-IOV Virtual Functions.
:param num_mdevcap: (int) The number of PCI devices capable of creating
mediated devices.
:param iommu_group: (int) Initial IOMMU group ID.
:param numa_node: (int) NUMA node of the device; if set all of the
devices will be assigned to the specified node else they will be
split between ``$TOTAL_NUMA_NODES`` nodes.
@ -439,19 +482,16 @@ class HostPCIDevicesInfo(object):
if num_pfs and num_vfs % num_pfs:
raise ValueError('num_vfs must be a factor of num_pfs')
slot = 0
bus = 0x81
slot = 0x0
function = 0
iommu_group = 40 # totally arbitrary number
# Generate PCI devs
for dev in range(num_pci):
pci_dev_name = self.pci_devname_template % {
'slot': slot, 'function': function}
LOG.info('Generating PCI device %r', pci_dev_name)
self.devices[pci_dev_name] = FakePCIDevice(
self.add_device(
dev_type='PCI',
bus=bus,
slot=slot,
function=function,
iommu_group=iommu_group,
@ -462,13 +502,9 @@ class HostPCIDevicesInfo(object):
# Generate MDEV capable devs
for dev in range(num_mdevcap):
pci_dev_name = self.pci_devname_template % {
'slot': slot, 'function': function}
LOG.info('Generating MDEV capable device %r', pci_dev_name)
self.devices[pci_dev_name] = FakePCIDevice(
self.add_device(
dev_type='MDEV_TYPES',
bus=bus,
slot=slot,
function=function,
iommu_group=iommu_group,
@ -485,19 +521,16 @@ class HostPCIDevicesInfo(object):
function = 0
numa_node_pf = self._calc_numa_node(dev, numa_node)
pci_dev_name = self.pci_devname_template % {
'slot': slot, 'function': function}
LOG.info('Generating PF device %r', pci_dev_name)
self.devices[pci_dev_name] = FakePCIDevice(
self.add_device(
dev_type='PF',
bus=bus,
slot=slot,
function=function,
iommu_group=iommu_group,
numa_node=numa_node_pf,
vf_ratio=vf_ratio)
pf_dev_name = pci_dev_name
parent = (bus, slot, function)
# Generate VFs
for _ in range(vf_ratio):
function += 1
@ -508,22 +541,46 @@ class HostPCIDevicesInfo(object):
slot += 1
function = 0
pci_dev_name = self.pci_devname_template % {
'slot': slot, 'function': function}
LOG.info('Generating VF device %r', pci_dev_name)
self.devices[pci_dev_name] = FakePCIDevice(
self.add_device(
dev_type='VF',
bus=bus,
slot=slot,
function=function,
iommu_group=iommu_group,
numa_node=numa_node_pf,
vf_ratio=vf_ratio,
parent=pf_dev_name)
parent=parent)
slot += 1
def add_device(
self, dev_type, bus, slot, function, iommu_group, numa_node,
vf_ratio=None, multiple_gpu_types=False, parent=None,
vend_id=None, vend_name=None, prod_id=None, prod_name=None,
driver_name=None,
):
pci_dev_name = _get_libvirt_nodedev_name(bus, slot, function)
LOG.info('Generating %s device %r', dev_type, pci_dev_name)
dev = FakePCIDevice(
dev_type=dev_type,
bus=bus,
slot=slot,
function=function,
iommu_group=iommu_group,
numa_node=numa_node,
vf_ratio=vf_ratio,
multiple_gpu_types=multiple_gpu_types,
parent=parent,
vend_id=vend_id,
vend_name=vend_name,
prod_id=prod_id,
prod_name=prod_name,
driver_name=driver_name)
self.devices[pci_dev_name] = dev
return dev
@classmethod
def _calc_numa_node(cls, dev, numa_node):
return dev % cls.TOTAL_NUMA_NODES if numa_node is None else numa_node
@ -581,6 +638,68 @@ class HostMdevDevicesInfo(object):
return dev
class FakeVDPADevice:
template = textwrap.dedent("""
<device>
<name>%(name)s</name>
<path>%(path)s</path>
<parent>%(parent)s</parent>
<driver>
<name>vhost_vdpa</name>
</driver>
<capability type='vdpa'>
<chardev>/dev/vhost-vdpa-%(idx)d</chardev>
</capability>
</device>""".strip())
def __init__(self, name, idx, parent):
assert isinstance(parent, FakePCIDevice)
assert parent.dev_type == 'VF'
self.name = name
self.idx = idx
self.parent = parent
self.generate_xml()
def generate_xml(self):
pf_pci = self.parent.parent
vf_pci = (self.parent.bus, self.parent.slot, self.parent.function)
pf_addr = '0000:%02x:%02x.%d' % pf_pci
vf_addr = '0000:%02x:%02x.%d' % vf_pci
parent = _get_libvirt_nodedev_name(*vf_pci)
path = f'/sys/devices/pci0000:00/{pf_addr}/{vf_addr}/vdpa{self.idx}'
self.xml = self.template % {
'name': self.name,
'idx': self.idx,
'path': path,
'parent': parent,
}
def XMLDesc(self, flags):
return self.xml
class HostVDPADevicesInfo:
def __init__(self):
self.devices = {}
def get_all_devices(self):
return self.devices.keys()
def get_device_by_name(self, device_name):
dev = self.devices[device_name]
return dev
def add_device(self, name, idx, parent):
LOG.info('Generating vDPA device %r', name)
dev = FakeVDPADevice(name=name, idx=idx, parent=parent)
self.devices[name] = dev
return dev
class HostInfo(object):
def __init__(self, cpu_nodes=1, cpu_sockets=1, cpu_cores=2, cpu_threads=1,
@ -994,6 +1113,8 @@ class Domain(object):
pci_bus, pci_slot,
pci_function)
nic_info['source'] = pci_device
elif nic_info['type'] == 'vdpa':
nic_info['source'] = source.get('dev')
nics_info += [nic_info]
@ -1184,24 +1305,31 @@ class Domain(object):
nics = ''
for nic in self._def['devices']['nics']:
if 'source' in nic and nic['type'] != 'hostdev':
nics += '''<interface type='%(type)s'>
<mac address='%(mac)s'/>
<source %(type)s='%(source)s'/>
<target dev='tap274487d1-60'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03'
function='0x0'/>
</interface>''' % nic
# this covers for direct nic type
else:
nics += '''<interface type='%(type)s'>
<mac address='%(mac)s'/>
<source>
<address type='pci' domain='0x0000' bus='0x81' slot='0x00'
function='0x01'/>
</source>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
</interface>''' % nic # noqa
if 'source' in nic:
if nic['type'] == 'hostdev':
nics += '''<interface type='%(type)s'>
<mac address='%(mac)s'/>
<source>
<address type='pci' domain='0x0000' bus='0x81' slot='0x00' function='0x01'/>
</source>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
</interface>''' % nic # noqa: E501
elif nic['type'] == 'vdpa':
# TODO(stephenfin): In real life, this would actually have
# an '<address>' element, but that requires information
# about the host that we're not passing through yet
nics += '''<interface type='%(type)s'>
<mac address='%(mac)s'/>
<source dev='%(source)s'/>
<model type='virtio'/>
</interface>'''
else:
nics += '''<interface type='%(type)s'>
<mac address='%(mac)s'/>
<source %(type)s='%(source)s'/>
<target dev='tap274487d1-60'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
</interface>''' % nic # noqa: E501
hostdevs = ''
for hostdev in self._def['devices']['hostdevs']:
@ -1458,9 +1586,11 @@ class Secret(object):
class Connection(object):
def __init__(self, uri=None, readonly=False, version=FAKE_LIBVIRT_VERSION,
hv_version=FAKE_QEMU_VERSION, host_info=None, pci_info=None,
mdev_info=None, hostname=None):
def __init__(
self, uri=None, readonly=False, version=FAKE_LIBVIRT_VERSION,
hv_version=FAKE_QEMU_VERSION, hostname=None,
host_info=None, pci_info=None, mdev_info=None, vdpa_info=None,
):
if not uri or uri == '':
if allow_default_uri_connection:
uri = 'qemu:///session'
@ -1498,6 +1628,7 @@ class Connection(object):
num_pfs=0,
num_vfs=0)
self.mdev_info = mdev_info or HostMdevDevicesInfo(devices={})
self.vdpa_info = vdpa_info or HostVDPADevicesInfo()
self.hostname = hostname or 'compute1'
def _add_nodedev(self, nodedev):
@ -1791,6 +1922,9 @@ class Connection(object):
if name.startswith('mdev'):
return self.mdev_info.get_device_by_name(name)
if name.startswith('vdpa'):
return self.vdpa_info.get_device_by_name(name)
pci_dev = self.pci_info.get_device_by_name(name)
if pci_dev:
return pci_dev
@ -1810,6 +1944,8 @@ class Connection(object):
return self.mdev_info.get_all_devices()
if cap == 'mdev_types':
return self.pci_info.get_all_mdev_capable_devices()
if cap == 'vdpa':
return self.vdpa_info.get_all_devices()
else:
raise ValueError('Capability "%s" is not supported' % cap)
@ -1843,11 +1979,22 @@ class Connection(object):
return secret
def listAllDevices(self, flags):
# Note this is incomplete as we do not filter
# based on the flags however it is enough for our
# current testing.
return [NodeDevice(self, xml=dev.XMLDesc(0))
for dev in self.pci_info.devices.values()]
devices = []
if flags & VIR_CONNECT_LIST_NODE_DEVICES_CAP_PCI_DEV:
devices.extend(
NodeDevice(self, xml=dev.XMLDesc(0))
for dev in self.pci_info.devices.values()
)
if flags & VIR_CONNECT_LIST_NODE_DEVICES_CAP_NET:
# TODO(stephenfin): Implement fake netdevs so we can test the
# capability reporting
pass
if flags & VIR_CONNECT_LIST_NODE_DEVICES_CAP_VDPA:
devices.extend(
NodeDevice(self, xml=dev.XMLDesc(0))
for dev in self.vdpa_info.devices.values()
)
return devices
def openAuth(uri, auth, flags=0):

3
nova/virt/libvirt/host.py

@ -1406,7 +1406,8 @@ class Host(object):
raise
def list_all_devices(
self, flags: int = 0) -> ty.List['libvirt.virNodeDevice']:
self, flags: int = 0,
) -> ty.List['libvirt.virNodeDevice']:
"""Lookup devices.
:param flags: a bitmask of flags to filter the returned devices.

Loading…
Cancel
Save