Merge "tests: Add functional test for vDPA device"
This commit is contained in:
Zuul
Gerrit Code Review
commit
70e7aff46e
|
|
@ -441,7 +441,7 @@ class PciDeviceStats(object):
|
|||
|
||||
if after_count < before_count:
|
||||
LOG.debug(
|
||||
'Dropped %d devices due to mismatched PCI attribute(s)',
|
||||
'Dropped %d device(s) due to mismatched PCI attribute(s)',
|
||||
before_count - after_count
|
||||
)
|
||||
|
||||
|
|
@ -458,7 +458,7 @@ class PciDeviceStats(object):
|
|||
|
||||
if after_count < before_count:
|
||||
LOG.debug(
|
||||
'Dropped %d devices as they are on the wrong NUMA node(s)',
|
||||
'Dropped %d device(s) as they are on the wrong NUMA node(s)',
|
||||
before_count - after_count
|
||||
)
|
||||
|
||||
|
|
@ -474,7 +474,7 @@ class PciDeviceStats(object):
|
|||
|
||||
if after_count < before_count:
|
||||
LOG.debug(
|
||||
'Dropped %d devices as they are PFs which we have not '
|
||||
'Dropped %d device(s) as they are PFs which we have not '
|
||||
'requested',
|
||||
before_count - after_count
|
||||
)
|
||||
|
|
@ -491,7 +491,7 @@ class PciDeviceStats(object):
|
|||
|
||||
if after_count < before_count:
|
||||
LOG.debug(
|
||||
'Dropped %d devices as they are VDPA devices which we have '
|
||||
'Dropped %d device(s) as they are VDPA devices which we have '
|
||||
'not requested',
|
||||
before_count - after_count
|
||||
)
|
||||
|
|
|
|||
|
|
@ -413,6 +413,24 @@ class InstanceHelperMixin:
|
|||
fake_notifier.wait_for_versioned_notifications('instance.reboot.end')
|
||||
return self._wait_for_state_change(server, expected_state)
|
||||
|
||||
def _attach_interface(self, server, port_uuid):
|
||||
"""attach a neutron port to a server."""
|
||||
body = {
|
||||
"interfaceAttachment": {
|
||||
"port_id": port_uuid
|
||||
}
|
||||
}
|
||||
attachment = self.api.attach_interface(server['id'], body)
|
||||
fake_notifier.wait_for_versioned_notifications(
|
||||
'instance.interface_attach.end')
|
||||
return attachment
|
||||
|
||||
def _detach_interface(self, server, port_uuid):
|
||||
"""detach a neutron port form a server."""
|
||||
self.api.detach_interface(server['id'], port_uuid)
|
||||
fake_notifier.wait_for_versioned_notifications(
|
||||
'instance.interface_detach.end')
|
||||
|
||||
def _rebuild_server(self, server, image_uuid, expected_state='ACTIVE'):
|
||||
"""Rebuild a server."""
|
||||
self.api.post_server_action(
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ class ServersTestBase(integrated_helpers._IntegratedTestBase):
|
|||
return self.start_service('scheduler')
|
||||
|
||||
def _get_connection(
|
||||
self, host_info=None, pci_info=None, mdev_info=None,
|
||||
self, host_info=None, pci_info=None, mdev_info=None, vdpa_info=None,
|
||||
libvirt_version=None, qemu_version=None, hostname=None,
|
||||
):
|
||||
if not host_info:
|
||||
|
|
@ -107,12 +107,14 @@ class ServersTestBase(integrated_helpers._IntegratedTestBase):
|
|||
host_info=host_info,
|
||||
pci_info=pci_info,
|
||||
mdev_info=mdev_info,
|
||||
vdpa_info=vdpa_info,
|
||||
hostname=hostname)
|
||||
return fake_connection
|
||||
|
||||
def start_compute(
|
||||
self, hostname='compute1', host_info=None, pci_info=None,
|
||||
mdev_info=None, libvirt_version=None, qemu_version=None,
|
||||
mdev_info=None, vdpa_info=None, libvirt_version=None,
|
||||
qemu_version=None,
|
||||
):
|
||||
"""Start a compute service.
|
||||
|
||||
|
|
@ -129,8 +131,8 @@ class ServersTestBase(integrated_helpers._IntegratedTestBase):
|
|||
|
||||
def _start_compute(hostname, host_info):
|
||||
fake_connection = self._get_connection(
|
||||
host_info, pci_info, mdev_info, libvirt_version, qemu_version,
|
||||
hostname,
|
||||
host_info, pci_info, mdev_info, vdpa_info, libvirt_version,
|
||||
qemu_version, hostname,
|
||||
)
|
||||
# This is fun. Firstly we need to do a global'ish mock so we can
|
||||
# actually start the service.
|
||||
|
|
@ -299,8 +301,8 @@ class LibvirtNeutronFixture(nova_fixtures.NeutronFixture):
|
|||
'subnet_id': subnet_4['id']
|
||||
}
|
||||
],
|
||||
'binding:vif_type': 'hw_veb',
|
||||
'binding:vif_details': {'vlan': 42},
|
||||
'binding:vif_type': 'hw_veb',
|
||||
'binding:vnic_type': 'direct',
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import mock
|
|||
from oslo_config import cfg
|
||||
from oslo_log import log as logging
|
||||
from oslo_serialization import jsonutils
|
||||
from oslo_utils.fixture import uuidsentinel as uuids
|
||||
from oslo_utils import units
|
||||
|
||||
import nova
|
||||
|
|
@ -275,9 +276,7 @@ class SRIOVServersTest(_PCIServersTestBase):
|
|||
self.assertNotIn('binding:profile', port)
|
||||
|
||||
# create a server using the VF via neutron
|
||||
flavor_id = self._create_flavor()
|
||||
self._create_server(
|
||||
flavor_id=flavor_id,
|
||||
networks=[
|
||||
{'port': base.LibvirtNeutronFixture.network_4_port_1['id']},
|
||||
],
|
||||
|
|
@ -548,9 +547,7 @@ class SRIOVServersTest(_PCIServersTestBase):
|
|||
self.neutron.create_port({'port': self.neutron.network_4_port_1})
|
||||
|
||||
# create a server using the VF via neutron
|
||||
flavor_id = self._create_flavor()
|
||||
self._create_server(
|
||||
flavor_id=flavor_id,
|
||||
networks=[
|
||||
{'port': base.LibvirtNeutronFixture.network_4_port_1['id']},
|
||||
],
|
||||
|
|
@ -672,6 +669,222 @@ class SRIOVAttachDetachTest(_PCIServersTestBase):
|
|||
self.neutron.sriov_pf_port2['id'])
|
||||
|
||||
|
||||
class VDPAServersTest(_PCIServersTestBase):
|
||||
|
||||
# this is needed for os_compute_api:os-migrate-server:migrate policy
|
||||
ADMIN_API = True
|
||||
microversion = 'latest'
|
||||
|
||||
# Whitelist both the PF and VF; in reality, you probably wouldn't do this
|
||||
# but we want to make sure that the PF is correctly taken off the table
|
||||
# once any VF is used
|
||||
PCI_PASSTHROUGH_WHITELIST = [jsonutils.dumps(x) for x in (
|
||||
{
|
||||
'vendor_id': '15b3',
|
||||
'product_id': '101d',
|
||||
'physical_network': 'physnet4',
|
||||
},
|
||||
{
|
||||
'vendor_id': '15b3',
|
||||
'product_id': '101e',
|
||||
'physical_network': 'physnet4',
|
||||
},
|
||||
)]
|
||||
# No need for aliases as these test will request SRIOV via neutron
|
||||
PCI_ALIAS = []
|
||||
|
||||
NUM_PFS = 1
|
||||
NUM_VFS = 4
|
||||
|
||||
FAKE_LIBVIRT_VERSION = 6_009_000 # 6.9.0
|
||||
FAKE_QEMU_VERSION = 5_001_000 # 5.1.0
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
|
||||
# The ultimate base class _IntegratedTestBase uses NeutronFixture but
|
||||
# we need a bit more intelligent neutron for these tests. Applying the
|
||||
# new fixture here means that we re-stub what the previous neutron
|
||||
# fixture already stubbed.
|
||||
self.neutron = self.useFixture(base.LibvirtNeutronFixture(self))
|
||||
|
||||
def start_compute(self):
|
||||
vf_ratio = self.NUM_VFS // self.NUM_PFS
|
||||
|
||||
pci_info = fakelibvirt.HostPCIDevicesInfo(
|
||||
num_pci=0, num_pfs=0, num_vfs=0)
|
||||
vdpa_info = fakelibvirt.HostVDPADevicesInfo()
|
||||
|
||||
pci_info.add_device(
|
||||
dev_type='PF',
|
||||
bus=0x6,
|
||||
slot=0x0,
|
||||
function=0,
|
||||
iommu_group=40, # totally arbitrary number
|
||||
numa_node=0,
|
||||
vf_ratio=vf_ratio,
|
||||
vend_id='15b3',
|
||||
vend_name='Mellanox Technologies',
|
||||
prod_id='101d',
|
||||
prod_name='MT2892 Family [ConnectX-6 Dx]',
|
||||
driver_name='mlx5_core')
|
||||
|
||||
for idx in range(self.NUM_VFS):
|
||||
vf = pci_info.add_device(
|
||||
dev_type='VF',
|
||||
bus=0x6,
|
||||
slot=0x0,
|
||||
function=idx + 1,
|
||||
iommu_group=idx + 41, # totally arbitrary number + offset
|
||||
numa_node=0,
|
||||
vf_ratio=vf_ratio,
|
||||
parent=(0x6, 0x0, 0),
|
||||
vend_id='15b3',
|
||||
vend_name='Mellanox Technologies',
|
||||
prod_id='101e',
|
||||
prod_name='ConnectX Family mlx5Gen Virtual Function',
|
||||
driver_name='mlx5_core')
|
||||
vdpa_info.add_device(f'vdpa_vdpa{idx}', idx, vf)
|
||||
|
||||
return super().start_compute(
|
||||
pci_info=pci_info, vdpa_info=vdpa_info,
|
||||
libvirt_version=self.FAKE_LIBVIRT_VERSION,
|
||||
qemu_version=self.FAKE_QEMU_VERSION)
|
||||
|
||||
def create_vdpa_port(self):
|
||||
vdpa_port = {
|
||||
'id': uuids.vdpa_port,
|
||||
'network_id': self.neutron.network_4['id'],
|
||||
'status': 'ACTIVE',
|
||||
'mac_address': 'b5:bc:2e:e7:51:ee',
|
||||
'fixed_ips': [
|
||||
{
|
||||
'ip_address': '192.168.4.6',
|
||||
'subnet_id': self.neutron.subnet_4['id']
|
||||
}
|
||||
],
|
||||
'binding:vif_details': {},
|
||||
'binding:vif_type': 'ovs',
|
||||
'binding:vnic_type': 'vdpa',
|
||||
}
|
||||
|
||||
# create the port
|
||||
self.neutron.create_port({'port': vdpa_port})
|
||||
return vdpa_port
|
||||
|
||||
def test_create_server(self):
|
||||
"""Create an instance using a neutron-provisioned vDPA VIF."""
|
||||
|
||||
orig_create = nova.virt.libvirt.guest.Guest.create
|
||||
|
||||
def fake_create(cls, xml, host):
|
||||
tree = etree.fromstring(xml)
|
||||
elem = tree.find('./devices/interface/[@type="vdpa"]')
|
||||
|
||||
# compare source device
|
||||
# the MAC address is derived from the neutron port, while the
|
||||
# source dev path assumes we attach vDPA devs in order
|
||||
expected = """
|
||||
<interface type="vdpa">
|
||||
<mac address="b5:bc:2e:e7:51:ee"/>
|
||||
<model type="virtio"/>
|
||||
<source dev="/dev/vhost-vdpa-3"/>
|
||||
</interface>"""
|
||||
actual = etree.tostring(elem, encoding='unicode')
|
||||
|
||||
self.assertXmlEqual(expected, actual)
|
||||
|
||||
return orig_create(xml, host)
|
||||
|
||||
self.stub_out(
|
||||
'nova.virt.libvirt.guest.Guest.create',
|
||||
fake_create,
|
||||
)
|
||||
|
||||
hostname = self.start_compute()
|
||||
num_pci = self.NUM_PFS + self.NUM_VFS
|
||||
|
||||
# both the PF and VF with vDPA capabilities (dev_type=vdpa) should have
|
||||
# been counted
|
||||
self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci)
|
||||
|
||||
# create the port
|
||||
vdpa_port = self.create_vdpa_port()
|
||||
|
||||
# ensure the binding details are currently unset
|
||||
port = self.neutron.show_port(vdpa_port['id'])['port']
|
||||
self.assertNotIn('binding:profile', port)
|
||||
|
||||
# create a server using the vDPA device via neutron
|
||||
self._create_server(networks=[{'port': vdpa_port['id']}])
|
||||
|
||||
# ensure there is one less VF available and that the PF is no longer
|
||||
# usable
|
||||
self.assertPCIDeviceCounts(hostname, total=num_pci, free=num_pci - 2)
|
||||
|
||||
# ensure the binding details sent to "neutron" were correct
|
||||
port = self.neutron.show_port(vdpa_port['id'])['port']
|
||||
self.assertIn('binding:profile', port)
|
||||
self.assertEqual(
|
||||
{
|
||||
'pci_vendor_info': '15b3:101e',
|
||||
'pci_slot': '0000:06:00.4',
|
||||
'physical_network': 'physnet4',
|
||||
},
|
||||
port['binding:profile'],
|
||||
)
|
||||
|
||||
def _test_common(self, op, *args, **kwargs):
|
||||
self.start_compute()
|
||||
|
||||
# create the port and a server, with the port attached to the server
|
||||
vdpa_port = self.create_vdpa_port()
|
||||
server = self._create_server(networks=[{'port': vdpa_port['id']}])
|
||||
|
||||
# attempt the unsupported action and ensure it fails
|
||||
ex = self.assertRaises(
|
||||
client.OpenStackApiException,
|
||||
op, server, *args, **kwargs)
|
||||
self.assertIn(
|
||||
'not supported for instance with vDPA ports',
|
||||
ex.response.text)
|
||||
|
||||
def test_attach_interface(self):
|
||||
self.start_compute()
|
||||
|
||||
# create the port and a server, but don't attach the port to the server
|
||||
# yet
|
||||
vdpa_port = self.create_vdpa_port()
|
||||
server = self._create_server(networks='none')
|
||||
|
||||
# attempt to attach the port to the server
|
||||
ex = self.assertRaises(
|
||||
client.OpenStackApiException,
|
||||
self._attach_interface, server, vdpa_port['id'])
|
||||
self.assertIn(
|
||||
'not supported for instance with vDPA ports',
|
||||
ex.response.text)
|
||||
|
||||
def test_detach_interface(self):
|
||||
self._test_common(self._detach_interface, uuids.vdpa_port)
|
||||
|
||||
def test_shelve(self):
|
||||
self._test_common(self._shelve_server)
|
||||
|
||||
def test_suspend(self):
|
||||
self._test_common(self._suspend_server)
|
||||
|
||||
def test_evacute(self):
|
||||
self._test_common(self._evacuate_server)
|
||||
|
||||
def test_resize(self):
|
||||
flavor_id = self._create_flavor()
|
||||
self._test_common(self._resize_server, flavor_id)
|
||||
|
||||
def test_cold_migrate(self):
|
||||
self._test_common(self._migrate_server)
|
||||
|
||||
|
||||
class PCIServersTest(_PCIServersTestBase):
|
||||
|
||||
ADMIN_API = True
|
||||
|
|
|
|||
|
|
@ -185,7 +185,7 @@ VIR_CONNECT_LIST_DOMAINS_INACTIVE = 2
|
|||
|
||||
# virConnectListAllNodeDevices flags
|
||||
VIR_CONNECT_LIST_NODE_DEVICES_CAP_PCI_DEV = 2
|
||||
VIR_CONNECT_LIST_NODE_DEVICES_CAP_NET = 16
|
||||
VIR_CONNECT_LIST_NODE_DEVICES_CAP_NET = 1 << 4
|
||||
VIR_CONNECT_LIST_NODE_DEVICES_CAP_VDPA = 1 << 17
|
||||
|
||||
# secret type
|
||||
|
|
@ -241,6 +241,11 @@ os_uname = collections.namedtuple(
|
|||
)
|
||||
|
||||
|
||||
def _get_libvirt_nodedev_name(bus, slot, function):
|
||||
"""Convert an address to a libvirt device name string."""
|
||||
return f'pci_0000_{bus:02x}_{slot:02x}_{function:d}'
|
||||
|
||||
|
||||
class FakePCIDevice(object):
|
||||
"""Generate a fake PCI device.
|
||||
|
||||
|
|
@ -255,22 +260,22 @@ class FakePCIDevice(object):
|
|||
pci_default_parent = "pci_0000_80_01_0"
|
||||
pci_device_template = textwrap.dedent("""
|
||||
<device>
|
||||
<name>pci_0000_81_%(slot)02x_%(function)d</name>
|
||||
<path>/sys/devices/pci0000:80/0000:80:01.0/0000:81:%(slot)02x.%(function)d</path>
|
||||
<name>pci_0000_%(bus)02x_%(slot)02x_%(function)d</name>
|
||||
<path>/sys/devices/pci0000:80/0000:80:01.0/0000:%(bus)02x:%(slot)02x.%(function)d</path>
|
||||
<parent>%(parent)s</parent>
|
||||
<driver>
|
||||
<name>%(driver)s</name>
|
||||
</driver>
|
||||
<capability type='pci'>
|
||||
<domain>0</domain>
|
||||
<bus>129</bus>
|
||||
<bus>%(bus)d</bus>
|
||||
<slot>%(slot)d</slot>
|
||||
<function>%(function)d</function>
|
||||
<product id='0x%(prod_id)s'>%(prod_name)s</product>
|
||||
<vendor id='0x%(vend_id)s'>%(vend_name)s</vendor>
|
||||
%(capability)s
|
||||
<iommuGroup number='%(iommu_group)d'>
|
||||
<address domain='0x0000' bus='0x81' slot='%(slot)#02x' function='0x%(function)d'/>
|
||||
<address domain='0x0000' bus='%(bus)#02x' slot='%(slot)#02x' function='0x%(function)d'/>
|
||||
</iommuGroup>
|
||||
<numa node='%(numa_node)s'/>
|
||||
<pci-express>
|
||||
|
|
@ -280,7 +285,7 @@ class FakePCIDevice(object):
|
|||
</capability>
|
||||
</device>""".strip()) # noqa
|
||||
cap_templ = "<capability type='%(cap_type)s'>%(addresses)s</capability>"
|
||||
addr_templ = "<address domain='0x0000' bus='0x81' slot='%(slot)#02x' function='%(function)#02x'/>" # noqa
|
||||
addr_templ = "<address domain='0x0000' bus='%(bus)#02x' slot='%(slot)#02x' function='%(function)#02x'/>" # noqa
|
||||
mdevtypes_templ = textwrap.dedent("""
|
||||
<type id='%(type_id)s'>
|
||||
<name>GRID M60-0B</name><deviceAPI>vfio-pci</deviceAPI>
|
||||
|
|
@ -289,22 +294,35 @@ class FakePCIDevice(object):
|
|||
|
||||
is_capable_of_mdevs = False
|
||||
|
||||
def __init__(self, dev_type, slot, function, iommu_group, numa_node,
|
||||
vf_ratio=None, multiple_gpu_types=False, parent=None):
|
||||
def __init__(
|
||||
self, dev_type, bus, slot, function, iommu_group, numa_node, *,
|
||||
vf_ratio=None, multiple_gpu_types=False, parent=None,
|
||||
vend_id=None, vend_name=None, prod_id=None, prod_name=None,
|
||||
driver_name=None,
|
||||
):
|
||||
"""Populate pci devices
|
||||
|
||||
:param dev_type: (string) Indicates the type of the device (PCI, PF,
|
||||
VF).
|
||||
:param dev_type: (str) Indicates the type of the device (PCI, PF, VF,
|
||||
MDEV_TYPES).
|
||||
:param bus: (int) Bus number of the device.
|
||||
:param slot: (int) Slot number of the device.
|
||||
:param function: (int) Function number of the device.
|
||||
:param iommu_group: (int) IOMMU group ID.
|
||||
:param numa_node: (int) NUMA node of the device.
|
||||
:param vf_ratio: (int) Ratio of Virtual Functions on Physical. Only
|
||||
applicable if ``dev_type`` is one of: ``PF``, ``VF``.
|
||||
:param multiple_gpu_types: (bool) Supports different vGPU types
|
||||
:param multiple_gpu_types: (bool) Supports different vGPU types.
|
||||
:param parent: (int, int, int) A tuple of bus, slot and function
|
||||
corresponding to the parent.
|
||||
:param vend_id: (str) The vendor ID.
|
||||
:param vend_name: (str) The vendor name.
|
||||
:param prod_id: (str) The product ID.
|
||||
:param prod_name: (str) The product name.
|
||||
:param driver_name: (str) The driver name.
|
||||
"""
|
||||
|
||||
self.dev_type = dev_type
|
||||
self.bus = bus
|
||||
self.slot = slot
|
||||
self.function = function
|
||||
self.iommu_group = iommu_group
|
||||
|
|
@ -312,28 +330,49 @@ class FakePCIDevice(object):
|
|||
self.vf_ratio = vf_ratio
|
||||
self.multiple_gpu_types = multiple_gpu_types
|
||||
self.parent = parent
|
||||
|
||||
self.vend_id = vend_id
|
||||
self.vend_name = vend_name
|
||||
self.prod_id = prod_id
|
||||
self.prod_name = prod_name
|
||||
self.driver_name = driver_name
|
||||
|
||||
self.generate_xml()
|
||||
|
||||
def generate_xml(self, skip_capability=False):
|
||||
vend_id = PCI_VEND_ID
|
||||
vend_name = PCI_VEND_NAME
|
||||
|
||||
# initial validation
|
||||
assert self.dev_type in ('PCI', 'VF', 'PF', 'MDEV_TYPES'), (
|
||||
f'got invalid dev_type {self.dev_type}')
|
||||
|
||||
if self.dev_type == 'PCI':
|
||||
assert not self.vf_ratio, 'vf_ratio does not apply for PCI devices'
|
||||
|
||||
if self.dev_type in ('PF', 'VF'):
|
||||
assert self.vf_ratio, 'require vf_ratio for PFs and VFs'
|
||||
|
||||
if self.dev_type == 'VF':
|
||||
assert self.parent, 'require parent for VFs'
|
||||
assert isinstance(self.parent, tuple), 'parent must be an address'
|
||||
assert len(self.parent) == 3, 'parent must be an address'
|
||||
|
||||
vend_id = self.vend_id or PCI_VEND_ID
|
||||
vend_name = self.vend_name or PCI_VEND_NAME
|
||||
capability = ''
|
||||
if self.dev_type == 'PCI':
|
||||
if self.vf_ratio:
|
||||
raise ValueError('vf_ratio does not apply for PCI devices')
|
||||
|
||||
prod_id = PCI_PROD_ID
|
||||
prod_name = PCI_PROD_NAME
|
||||
driver = PCI_DRIVER_NAME
|
||||
prod_id = self.prod_id or PCI_PROD_ID
|
||||
prod_name = self.prod_name or PCI_PROD_NAME
|
||||
driver = self.driver_name or PCI_DRIVER_NAME
|
||||
elif self.dev_type == 'PF':
|
||||
prod_id = PF_PROD_ID
|
||||
prod_name = PF_PROD_NAME
|
||||
driver = PF_DRIVER_NAME
|
||||
prod_id = self.prod_id or PF_PROD_ID
|
||||
prod_name = self.prod_name or PF_PROD_NAME
|
||||
driver = self.driver_name or PF_DRIVER_NAME
|
||||
if not skip_capability:
|
||||
capability = self.cap_templ % {
|
||||
'cap_type': PF_CAP_TYPE,
|
||||
'addresses': '\n'.join([
|
||||
self.addr_templ % {
|
||||
'bus': self.bus,
|
||||
# these are the slot, function values of the child
|
||||
# VFs, we can only assign 8 functions to a slot
|
||||
# (0-7) so bump the slot each time we exceed this
|
||||
|
|
@ -344,13 +383,14 @@ class FakePCIDevice(object):
|
|||
} for x in range(1, self.vf_ratio + 1)])
|
||||
}
|
||||
elif self.dev_type == 'VF':
|
||||
prod_id = VF_PROD_ID
|
||||
prod_name = VF_PROD_NAME
|
||||
driver = VF_DRIVER_NAME
|
||||
prod_id = self.prod_id or VF_PROD_ID
|
||||
prod_name = self.prod_name or VF_PROD_NAME
|
||||
driver = self.driver_name or VF_DRIVER_NAME
|
||||
if not skip_capability:
|
||||
capability = self.cap_templ % {
|
||||
'cap_type': VF_CAP_TYPE,
|
||||
'addresses': self.addr_templ % {
|
||||
'bus': self.bus,
|
||||
# this is the slot, function value of the parent PF
|
||||
# if we're e.g. device 8, we'll have a different slot
|
||||
# to our parent so reverse this
|
||||
|
|
@ -360,11 +400,11 @@ class FakePCIDevice(object):
|
|||
}
|
||||
}
|
||||
elif self.dev_type == 'MDEV_TYPES':
|
||||
prod_id = MDEV_CAPABLE_PROD_ID
|
||||
prod_name = MDEV_CAPABLE_PROD_NAME
|
||||
driver = MDEV_CAPABLE_DRIVER_NAME
|
||||
vend_id = MDEV_CAPABLE_VEND_ID
|
||||
vend_name = MDEV_CAPABLE_VEND_NAME
|
||||
prod_id = self.prod_id or MDEV_CAPABLE_PROD_ID
|
||||
prod_name = self.prod_name or MDEV_CAPABLE_PROD_NAME
|
||||
driver = self.driver_name or MDEV_CAPABLE_DRIVER_NAME
|
||||
vend_id = self.vend_id or MDEV_CAPABLE_VEND_ID
|
||||
vend_name = self.vend_name or MDEV_CAPABLE_VEND_NAME
|
||||
types = [self.mdevtypes_templ % {
|
||||
'type_id': NVIDIA_11_VGPU_TYPE,
|
||||
'instances': 16,
|
||||
|
|
@ -380,10 +420,13 @@ class FakePCIDevice(object):
|
|||
'addresses': '\n'.join(types)
|
||||
}
|
||||
self.is_capable_of_mdevs = True
|
||||
else:
|
||||
raise ValueError('Expected one of: PCI, VF, PCI')
|
||||
|
||||
parent = self.pci_default_parent
|
||||
if self.parent:
|
||||
parent = _get_libvirt_nodedev_name(*self.parent)
|
||||
|
||||
self.pci_device = self.pci_device_template % {
|
||||
'bus': self.bus,
|
||||
'slot': self.slot,
|
||||
'function': self.function,
|
||||
'vend_id': vend_id,
|
||||
|
|
@ -394,7 +437,7 @@ class FakePCIDevice(object):
|
|||
'capability': capability,
|
||||
'iommu_group': self.iommu_group,
|
||||
'numa_node': self.numa_node,
|
||||
'parent': self.parent or self.pci_default_parent
|
||||
'parent': parent,
|
||||
}
|
||||
# -1 is the sentinel set in /sys/bus/pci/devices/*/numa_node
|
||||
# for no NUMA affinity. When the numa_node is set to -1 on a device
|
||||
|
|
@ -406,11 +449,12 @@ class FakePCIDevice(object):
|
|||
return self.pci_device
|
||||
|
||||
|
||||
# TODO(stephenfin): Remove all of these HostFooDevicesInfo objects in favour of
|
||||
# a unified devices object
|
||||
class HostPCIDevicesInfo(object):
|
||||
"""Represent a pool of host PCI devices."""
|
||||
|
||||
TOTAL_NUMA_NODES = 2
|
||||
pci_devname_template = 'pci_0000_81_%(slot)02x_%(function)d'
|
||||
|
||||
def __init__(self, num_pci=0, num_pfs=2, num_vfs=8, num_mdevcap=0,
|
||||
numa_node=None, multiple_gpu_types=False):
|
||||
|
|
@ -422,7 +466,6 @@ class HostPCIDevicesInfo(object):
|
|||
:param num_vfs: (int) The number of PCI SR-IOV Virtual Functions.
|
||||
:param num_mdevcap: (int) The number of PCI devices capable of creating
|
||||
mediated devices.
|
||||
:param iommu_group: (int) Initial IOMMU group ID.
|
||||
:param numa_node: (int) NUMA node of the device; if set all of the
|
||||
devices will be assigned to the specified node else they will be
|
||||
split between ``$TOTAL_NUMA_NODES`` nodes.
|
||||
|
|
@ -439,19 +482,16 @@ class HostPCIDevicesInfo(object):
|
|||
if num_pfs and num_vfs % num_pfs:
|
||||
raise ValueError('num_vfs must be a factor of num_pfs')
|
||||
|
||||
slot = 0
|
||||
bus = 0x81
|
||||
slot = 0x0
|
||||
function = 0
|
||||
iommu_group = 40 # totally arbitrary number
|
||||
|
||||
# Generate PCI devs
|
||||
for dev in range(num_pci):
|
||||
pci_dev_name = self.pci_devname_template % {
|
||||
'slot': slot, 'function': function}
|
||||
|
||||
LOG.info('Generating PCI device %r', pci_dev_name)
|
||||
|
||||
self.devices[pci_dev_name] = FakePCIDevice(
|
||||
self.add_device(
|
||||
dev_type='PCI',
|
||||
bus=bus,
|
||||
slot=slot,
|
||||
function=function,
|
||||
iommu_group=iommu_group,
|
||||
|
|
@ -462,13 +502,9 @@ class HostPCIDevicesInfo(object):
|
|||
|
||||
# Generate MDEV capable devs
|
||||
for dev in range(num_mdevcap):
|
||||
pci_dev_name = self.pci_devname_template % {
|
||||
'slot': slot, 'function': function}
|
||||
|
||||
LOG.info('Generating MDEV capable device %r', pci_dev_name)
|
||||
|
||||
self.devices[pci_dev_name] = FakePCIDevice(
|
||||
self.add_device(
|
||||
dev_type='MDEV_TYPES',
|
||||
bus=bus,
|
||||
slot=slot,
|
||||
function=function,
|
||||
iommu_group=iommu_group,
|
||||
|
|
@ -485,19 +521,16 @@ class HostPCIDevicesInfo(object):
|
|||
function = 0
|
||||
numa_node_pf = self._calc_numa_node(dev, numa_node)
|
||||
|
||||
pci_dev_name = self.pci_devname_template % {
|
||||
'slot': slot, 'function': function}
|
||||
|
||||
LOG.info('Generating PF device %r', pci_dev_name)
|
||||
|
||||
self.devices[pci_dev_name] = FakePCIDevice(
|
||||
self.add_device(
|
||||
dev_type='PF',
|
||||
bus=bus,
|
||||
slot=slot,
|
||||
function=function,
|
||||
iommu_group=iommu_group,
|
||||
numa_node=numa_node_pf,
|
||||
vf_ratio=vf_ratio)
|
||||
pf_dev_name = pci_dev_name
|
||||
|
||||
parent = (bus, slot, function)
|
||||
# Generate VFs
|
||||
for _ in range(vf_ratio):
|
||||
function += 1
|
||||
|
|
@ -508,22 +541,46 @@ class HostPCIDevicesInfo(object):
|
|||
slot += 1
|
||||
function = 0
|
||||
|
||||
pci_dev_name = self.pci_devname_template % {
|
||||
'slot': slot, 'function': function}
|
||||
|
||||
LOG.info('Generating VF device %r', pci_dev_name)
|
||||
|
||||
self.devices[pci_dev_name] = FakePCIDevice(
|
||||
self.add_device(
|
||||
dev_type='VF',
|
||||
bus=bus,
|
||||
slot=slot,
|
||||
function=function,
|
||||
iommu_group=iommu_group,
|
||||
numa_node=numa_node_pf,
|
||||
vf_ratio=vf_ratio,
|
||||
parent=pf_dev_name)
|
||||
parent=parent)
|
||||
|
||||
slot += 1
|
||||
|
||||
def add_device(
|
||||
self, dev_type, bus, slot, function, iommu_group, numa_node,
|
||||
vf_ratio=None, multiple_gpu_types=False, parent=None,
|
||||
vend_id=None, vend_name=None, prod_id=None, prod_name=None,
|
||||
driver_name=None,
|
||||
):
|
||||
pci_dev_name = _get_libvirt_nodedev_name(bus, slot, function)
|
||||
|
||||
LOG.info('Generating %s device %r', dev_type, pci_dev_name)
|
||||
|
||||
dev = FakePCIDevice(
|
||||
dev_type=dev_type,
|
||||
bus=bus,
|
||||
slot=slot,
|
||||
function=function,
|
||||
iommu_group=iommu_group,
|
||||
numa_node=numa_node,
|
||||
vf_ratio=vf_ratio,
|
||||
multiple_gpu_types=multiple_gpu_types,
|
||||
parent=parent,
|
||||
vend_id=vend_id,
|
||||
vend_name=vend_name,
|
||||
prod_id=prod_id,
|
||||
prod_name=prod_name,
|
||||
driver_name=driver_name)
|
||||
self.devices[pci_dev_name] = dev
|
||||
return dev
|
||||
|
||||
@classmethod
|
||||
def _calc_numa_node(cls, dev, numa_node):
|
||||
return dev % cls.TOTAL_NUMA_NODES if numa_node is None else numa_node
|
||||
|
|
@ -581,6 +638,68 @@ class HostMdevDevicesInfo(object):
|
|||
return dev
|
||||
|
||||
|
||||
class FakeVDPADevice:
|
||||
|
||||
template = textwrap.dedent("""
|
||||
<device>
|
||||
<name>%(name)s</name>
|
||||
<path>%(path)s</path>
|
||||
<parent>%(parent)s</parent>
|
||||
<driver>
|
||||
<name>vhost_vdpa</name>
|
||||
</driver>
|
||||
<capability type='vdpa'>
|
||||
<chardev>/dev/vhost-vdpa-%(idx)d</chardev>
|
||||
</capability>
|
||||
</device>""".strip())
|
||||
|
||||
def __init__(self, name, idx, parent):
|
||||
assert isinstance(parent, FakePCIDevice)
|
||||
assert parent.dev_type == 'VF'
|
||||
|
||||
self.name = name
|
||||
self.idx = idx
|
||||
self.parent = parent
|
||||
self.generate_xml()
|
||||
|
||||
def generate_xml(self):
|
||||
pf_pci = self.parent.parent
|
||||
vf_pci = (self.parent.bus, self.parent.slot, self.parent.function)
|
||||
pf_addr = '0000:%02x:%02x.%d' % pf_pci
|
||||
vf_addr = '0000:%02x:%02x.%d' % vf_pci
|
||||
parent = _get_libvirt_nodedev_name(*vf_pci)
|
||||
path = f'/sys/devices/pci0000:00/{pf_addr}/{vf_addr}/vdpa{self.idx}'
|
||||
self.xml = self.template % {
|
||||
'name': self.name,
|
||||
'idx': self.idx,
|
||||
'path': path,
|
||||
'parent': parent,
|
||||
}
|
||||
|
||||
def XMLDesc(self, flags):
|
||||
return self.xml
|
||||
|
||||
|
||||
class HostVDPADevicesInfo:
|
||||
|
||||
def __init__(self):
|
||||
self.devices = {}
|
||||
|
||||
def get_all_devices(self):
|
||||
return self.devices.keys()
|
||||
|
||||
def get_device_by_name(self, device_name):
|
||||
dev = self.devices[device_name]
|
||||
return dev
|
||||
|
||||
def add_device(self, name, idx, parent):
|
||||
LOG.info('Generating vDPA device %r', name)
|
||||
|
||||
dev = FakeVDPADevice(name=name, idx=idx, parent=parent)
|
||||
self.devices[name] = dev
|
||||
return dev
|
||||
|
||||
|
||||
class HostInfo(object):
|
||||
|
||||
def __init__(self, cpu_nodes=1, cpu_sockets=1, cpu_cores=2, cpu_threads=1,
|
||||
|
|
@ -994,6 +1113,8 @@ class Domain(object):
|
|||
pci_bus, pci_slot,
|
||||
pci_function)
|
||||
nic_info['source'] = pci_device
|
||||
elif nic_info['type'] == 'vdpa':
|
||||
nic_info['source'] = source.get('dev')
|
||||
|
||||
nics_info += [nic_info]
|
||||
|
||||
|
|
@ -1184,24 +1305,31 @@ class Domain(object):
|
|||
|
||||
nics = ''
|
||||
for nic in self._def['devices']['nics']:
|
||||
if 'source' in nic and nic['type'] != 'hostdev':
|
||||
if 'source' in nic:
|
||||
if nic['type'] == 'hostdev':
|
||||
nics += '''<interface type='%(type)s'>
|
||||
<mac address='%(mac)s'/>
|
||||
<source>
|
||||
<address type='pci' domain='0x0000' bus='0x81' slot='0x00' function='0x01'/>
|
||||
</source>
|
||||
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
|
||||
</interface>''' % nic # noqa: E501
|
||||
elif nic['type'] == 'vdpa':
|
||||
# TODO(stephenfin): In real life, this would actually have
|
||||
# an '<address>' element, but that requires information
|
||||
# about the host that we're not passing through yet
|
||||
nics += '''<interface type='%(type)s'>
|
||||
<mac address='%(mac)s'/>
|
||||
<source dev='%(source)s'/>
|
||||
<model type='virtio'/>
|
||||
</interface>'''
|
||||
else:
|
||||
nics += '''<interface type='%(type)s'>
|
||||
<mac address='%(mac)s'/>
|
||||
<source %(type)s='%(source)s'/>
|
||||
<target dev='tap274487d1-60'/>
|
||||
<address type='pci' domain='0x0000' bus='0x00' slot='0x03'
|
||||
function='0x0'/>
|
||||
</interface>''' % nic
|
||||
# this covers for direct nic type
|
||||
else:
|
||||
nics += '''<interface type='%(type)s'>
|
||||
<mac address='%(mac)s'/>
|
||||
<source>
|
||||
<address type='pci' domain='0x0000' bus='0x81' slot='0x00'
|
||||
function='0x01'/>
|
||||
</source>
|
||||
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
|
||||
</interface>''' % nic # noqa
|
||||
</interface>''' % nic # noqa: E501
|
||||
|
||||
hostdevs = ''
|
||||
for hostdev in self._def['devices']['hostdevs']:
|
||||
|
|
@ -1458,9 +1586,11 @@ class Secret(object):
|
|||
|
||||
|
||||
class Connection(object):
|
||||
def __init__(self, uri=None, readonly=False, version=FAKE_LIBVIRT_VERSION,
|
||||
hv_version=FAKE_QEMU_VERSION, host_info=None, pci_info=None,
|
||||
mdev_info=None, hostname=None):
|
||||
def __init__(
|
||||
self, uri=None, readonly=False, version=FAKE_LIBVIRT_VERSION,
|
||||
hv_version=FAKE_QEMU_VERSION, hostname=None,
|
||||
host_info=None, pci_info=None, mdev_info=None, vdpa_info=None,
|
||||
):
|
||||
if not uri or uri == '':
|
||||
if allow_default_uri_connection:
|
||||
uri = 'qemu:///session'
|
||||
|
|
@ -1498,6 +1628,7 @@ class Connection(object):
|
|||
num_pfs=0,
|
||||
num_vfs=0)
|
||||
self.mdev_info = mdev_info or HostMdevDevicesInfo(devices={})
|
||||
self.vdpa_info = vdpa_info or HostVDPADevicesInfo()
|
||||
self.hostname = hostname or 'compute1'
|
||||
|
||||
def _add_nodedev(self, nodedev):
|
||||
|
|
@ -1791,6 +1922,9 @@ class Connection(object):
|
|||
if name.startswith('mdev'):
|
||||
return self.mdev_info.get_device_by_name(name)
|
||||
|
||||
if name.startswith('vdpa'):
|
||||
return self.vdpa_info.get_device_by_name(name)
|
||||
|
||||
pci_dev = self.pci_info.get_device_by_name(name)
|
||||
if pci_dev:
|
||||
return pci_dev
|
||||
|
|
@ -1810,6 +1944,8 @@ class Connection(object):
|
|||
return self.mdev_info.get_all_devices()
|
||||
if cap == 'mdev_types':
|
||||
return self.pci_info.get_all_mdev_capable_devices()
|
||||
if cap == 'vdpa':
|
||||
return self.vdpa_info.get_all_devices()
|
||||
else:
|
||||
raise ValueError('Capability "%s" is not supported' % cap)
|
||||
|
||||
|
|
@ -1843,11 +1979,22 @@ class Connection(object):
|
|||
return secret
|
||||
|
||||
def listAllDevices(self, flags):
|
||||
# Note this is incomplete as we do not filter
|
||||
# based on the flags however it is enough for our
|
||||
# current testing.
|
||||
return [NodeDevice(self, xml=dev.XMLDesc(0))
|
||||
for dev in self.pci_info.devices.values()]
|
||||
devices = []
|
||||
if flags & VIR_CONNECT_LIST_NODE_DEVICES_CAP_PCI_DEV:
|
||||
devices.extend(
|
||||
NodeDevice(self, xml=dev.XMLDesc(0))
|
||||
for dev in self.pci_info.devices.values()
|
||||
)
|
||||
if flags & VIR_CONNECT_LIST_NODE_DEVICES_CAP_NET:
|
||||
# TODO(stephenfin): Implement fake netdevs so we can test the
|
||||
# capability reporting
|
||||
pass
|
||||
if flags & VIR_CONNECT_LIST_NODE_DEVICES_CAP_VDPA:
|
||||
devices.extend(
|
||||
NodeDevice(self, xml=dev.XMLDesc(0))
|
||||
for dev in self.vdpa_info.devices.values()
|
||||
)
|
||||
return devices
|
||||
|
||||
|
||||
def openAuth(uri, auth, flags=0):
|
||||
|
|
|
|||
|
|
@ -1406,7 +1406,8 @@ class Host(object):
|
|||
raise
|
||||
|
||||
def list_all_devices(
|
||||
self, flags: int = 0) -> ty.List['libvirt.virNodeDevice']:
|
||||
self, flags: int = 0,
|
||||
) -> ty.List['libvirt.virNodeDevice']:
|
||||
"""Lookup devices.
|
||||
|
||||
:param flags: a bitmask of flags to filter the returned devices.
|
||||
|
|
|
|||
Loading…
Reference in New Issue