openstack
/
nova
Code Issues Proposed changes

Merge "libvirt: Add vDPA nodedev parsing"

This commit is contained in:
Zuul 2021-03-15 13:38:20 +00:00 committed by Gerrit Code Review
parent 20fdd3c0bc 9a673a8faa
commit 896bf25e74
6 changed files with 243 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
nova/tests/unit/virt/libvirt/fakelibvirt.py
View File

@ -185,6 +185,7 @@ VIR_CONNECT_LIST_DOMAINS_INACTIVE = 2
# virConnectListAllNodeDevices flags
VIR_CONNECT_LIST_NODE_DEVICES_CAP_PCI_DEV = 2
VIR_CONNECT_LIST_NODE_DEVICES_CAP_NET = 16
VIR_CONNECT_LIST_NODE_DEVICES_CAP_VDPA = 1 << 17
# secret type
VIR_SECRET_USAGE_TYPE_NONE = 0

20
nova/tests/unit/virt/libvirt/test_config.py
View File

@ -3118,6 +3118,26 @@ class LibvirtConfigNodeDeviceTest(LibvirtConfigBaseTest):
self.assertEqual("nvidia-11", obj.mdev_information.type)
self.assertEqual(12, obj.mdev_information.iommu_group)
def test_config_vdpa_device(self):
xmlin = """
<device>
<name>vdpa_vdpa0</name>
<path>/sys/devices/pci0000:00/0000:00:02.2/0000:06:00.2/vdpa0</path>
<parent>pci_0000_06_00_2</parent>
<driver>
<name>vhost_vdpa</name>
</driver>
<capability type='vdpa'>
<chardev>/dev/vhost-vdpa-0</chardev>
</capability>
</device>"""
obj = config.LibvirtConfigNodeDevice()
obj.parse_str(xmlin)
self.assertIsInstance(
obj.vdpa_capability, config.LibvirtConfigNodeDeviceVDPACap)
self.assertEqual("/dev/vhost-vdpa-0", obj.vdpa_capability.dev_path)
class LibvirtConfigNodeDevicePciCapTest(LibvirtConfigBaseTest):

135
nova/tests/unit/virt/libvirt/test_host.py
View File

@ -1101,7 +1101,7 @@ Active: 8381604 kB
pci_dev = fakelibvirt.NodeDevice(
self.host._get_connection(),
xml=fake_libvirt_data._fake_NodeDevXml[dev_name])
actual_vf = self.host._get_pcidev_info(dev_name, pci_dev, [])
actual_vf = self.host._get_pcidev_info(dev_name, pci_dev, [], [])
expect_vf = {
"dev_id": dev_name, "address": "0000:04:11.7",
"product_id": '1520', "numa_node": 0,
@ -1134,7 +1134,8 @@ Active: 8381604 kB
dev for dev in node_devs.values() if dev.name() not in devs]
name = "pci_0000_04_00_3"
actual_vf = self.host._get_pcidev_info(name, node_devs[name], net_devs)
actual_vf = self.host._get_pcidev_info(
name, node_devs[name], net_devs, [])
expect_vf = {
"dev_id": "pci_0000_04_00_3",
"address": "0000:04:00.3",
@ -1147,7 +1148,8 @@ Active: 8381604 kB
self.assertEqual(expect_vf, actual_vf)
name = "pci_0000_04_10_7"
actual_vf = self.host._get_pcidev_info(name, node_devs[name], net_devs)
actual_vf = self.host._get_pcidev_info(
name, node_devs[name], net_devs, [])
expect_vf = {
"dev_id": "pci_0000_04_10_7",
"address": "0000:04:10.7",
@ -1165,7 +1167,8 @@ Active: 8381604 kB
self.assertEqual(expect_vf, actual_vf)
name = "pci_0000_04_11_7"
actual_vf = self.host._get_pcidev_info(name, node_devs[name], net_devs)
actual_vf = self.host._get_pcidev_info(
name, node_devs[name], net_devs, [])
expect_vf = {
"dev_id": "pci_0000_04_11_7",
"address": "0000:04:11.7",
@ -1183,7 +1186,8 @@ Active: 8381604 kB
self.assertEqual(expect_vf, actual_vf)
name = "pci_0000_04_00_1"
actual_vf = self.host._get_pcidev_info(name, node_devs[name], net_devs)
actual_vf = self.host._get_pcidev_info(
name, node_devs[name], net_devs, [])
expect_vf = {
"dev_id": "pci_0000_04_00_1",
"address": "0000:04:00.1",
@ -1196,7 +1200,8 @@ Active: 8381604 kB
self.assertEqual(expect_vf, actual_vf)
name = "pci_0000_03_00_0"
actual_vf = self.host._get_pcidev_info(name, node_devs[name], net_devs)
actual_vf = self.host._get_pcidev_info(
name, node_devs[name], net_devs, [])
expect_vf = {
"dev_id": "pci_0000_03_00_0",
"address": "0000:03:00.0",
@ -1209,7 +1214,8 @@ Active: 8381604 kB
self.assertEqual(expect_vf, actual_vf)
name = "pci_0000_03_00_1"
actual_vf = self.host._get_pcidev_info(name, node_devs[name], net_devs)
actual_vf = self.host._get_pcidev_info(
name, node_devs[name], net_devs, [])
expect_vf = {
"dev_id": "pci_0000_03_00_1",
"address": "0000:03:00.1",
@ -1304,6 +1310,121 @@ Active: 8381604 kB
self.assertEqual([], ret)
mock_list_all_devices.assert_called_once_with(42)
def test_get_vdpa_nodedev_by_address(self):
with test.nested(
mock.patch.object(
self.host.get_connection(), "listAllDevices"),
mock.patch.object(self.host, "_get_pcinet_info"),
) as (mock_list_all_devices, mock_get_pci_info):
vdpa_str = """
<device>
<name>vdpa_vdpa0</name>
<path>/sys/devices/pci0000:00/0000:00:02.2/0000:06:00.2/vdpa0</path>
<parent>pci_0000_06_00_2</parent>
<driver>
<name>vhost_vdpa</name>
</driver>
<capability type='vdpa'>
<chardev>/dev/vhost-vdpa-0</chardev>
</capability>
</device>""" # noqa: E501
vdpa_dev = fakelibvirt.NodeDevice(None, xml=vdpa_str)
vf_str = """
<device>
<name>pci_0000_06_00_2</name>
<path>/sys/devices/pci0000:00/0000:00:02.2/0000:06:00.2</path>
<parent>pci_0000_00_02_2</parent>
<driver>
<name>mlx5_core</name>
</driver>
<capability type='pci'>
<class>0x020000</class>
<domain>0</domain>
<bus>6</bus>
<slot>0</slot>
<function>2</function>
<product id='0x101e'>ConnectX Family mlx5Gen Virtual Function</product>
<vendor id='0x15b3'>Mellanox Technologies</vendor>
<capability type='phys_function'>
<address domain='0x0000' bus='0x06' slot='0x00' function='0x0'/>
</capability>
<iommuGroup number='99'>
<address domain='0x0000' bus='0x06' slot='0x00' function='0x2'/>
</iommuGroup>
<numa node='0'/>
<pci-express>
<link validity='cap' port='0' speed='16' width='8'/>
<link validity='sta' width='0'/>
</pci-express>
</capability>
</device>""" # noqa: E501
vf_dev = fakelibvirt.NodeDevice(None, xml=vf_str)
node_devs = [vdpa_dev, vf_dev]
mock_list_all_devices.return_value = node_devs
mock_get_pci_info.return_value = [
{
"dev_id": "pci_0000_06_00_2",
"address": "0000:06:00.2",
"product_id": '101e',
"vendor_id": '15b3',
"numa_node": 0,
"label": 'label_101e_15b3',
"dev_type": 'type-VF',
"parent_addr": "0000:00:02.2",
"parent_ifname": "enp6s0f0",
"capabilities": {
"network": [
"rx", "tx", "sg", "tso", "gso", "gro", "rxvlan",
"txvlan",
],
},
},
{
"dev_id": "vdpa_vdpa0",
"address": "0000:06:00.2",
"product_id": '101e',
"vendor_id": '15b3',
"numa_node": 0,
"label": 'label_101e_15b3',
"dev_type": 'vdpa',
'parent_addr': '0000:06:00.2',
"parent_ifname": "enp6s0f0v0",
"capabilities": {
"network": [
"rx", "tx", "sg", "tso", "gso", "gro",
"rxvlan", "txvlan"],
},
},
]
ret = self.host.get_vdpa_nodedev_by_address("0000:06:00.2")
cfgdev = vconfig.LibvirtConfigNodeDevice()
cfgdev.parse_str(vdpa_str)
self.assertEqual(cfgdev.name, ret.name)
self.assertEqual(
"/dev/vhost-vdpa-0", ret.vdpa_capability.dev_path)
def test_get_vdpa_device_path(self):
with mock.patch.object(
self.host, "get_vdpa_nodedev_by_address",
) as mock_vdpa:
xml_str = """
<device>
<name>vdpa_vdpa0</name>
<path>/sys/devices/pci0000:00/0000:00:02.2/0000:06:00.2/vdpa0</path>
<parent>pci_0000_06_00_2</parent>
<driver>
<name>vhost_vdpa</name>
</driver>
<capability type='vdpa'>
<chardev>/dev/vhost-vdpa-0</chardev>
</capability>
</device>"""
cfgdev = vconfig.LibvirtConfigNodeDevice()
cfgdev.parse_str(xml_str)
mock_vdpa.return_value = cfgdev
ret = self.host.get_vdpa_device_path("0000:06:00.2")
self.assertEqual("/dev/vhost-vdpa-0", ret)
@mock.patch.object(fakelibvirt.virConnect, "listDevices")
def test_list_devices(self, mock_listDevices):
self.host._list_devices('mdev', 8)

22
nova/virt/libvirt/config.py
View File

@ -3103,6 +3103,7 @@ class LibvirtConfigNodeDevice(LibvirtConfigObject):
self.parent = None
self.pci_capability = None
self.mdev_information = None
self.vdpa_capability = None
def parse_dom(self, xmldoc):
super(LibvirtConfigNodeDevice, self).parse_dom(xmldoc)
@ -3120,6 +3121,23 @@ class LibvirtConfigNodeDevice(LibvirtConfigObject):
mdev_info = LibvirtConfigNodeDeviceMdevInformation()
mdev_info.parse_dom(c)
self.mdev_information = mdev_info
elif c.tag == "capability" and c.get("type") in ['vdpa']:
vdpa_caps = LibvirtConfigNodeDeviceVDPACap()
vdpa_caps.parse_dom(c)
self.vdpa_capability = vdpa_caps
class LibvirtConfigNodeDeviceVDPACap(LibvirtConfigObject):
def __init__(self, **kwargs):
super().__init__(
root_name="capability", **kwargs)
self.dev_path = None
def parse_dom(self, xmldoc):
super().parse_dom(xmldoc)
for c in xmldoc:
if c.tag == "chardev":
self.dev_path = c.text
class LibvirtConfigNodeDevicePciCap(LibvirtConfigObject):
@ -3182,6 +3200,10 @@ class LibvirtConfigNodeDevicePciCap(LibvirtConfigObject):
mdevcap.parse_dom(c)
self.mdev_capability.append(mdevcap)
def pci_address(self):
return "%04x:%02x:%02x.%01x" % (
self.domain, self.bus, self.slot, self.function)
class LibvirtConfigNodeDevicePciSubFunctionCap(LibvirtConfigObject):
def __init__(self, **kwargs):

21
nova/virt/libvirt/driver.py
View File

@ -242,6 +242,10 @@ VGPU_RESOURCE_SEMAPHORE = 'vgpu_resources'
LIBVIRT_PERF_EVENT_PREFIX = 'VIR_PERF_PARAM_'
# VDPA interface support
MIN_LIBVIRT_VDPA = (6, 9, 0)
MIN_QEMU_VDPA = (5, 1, 0)
class AsyncDeviceEventsHandler:
"""A synchornization point between libvirt events an clients waiting for
@ -7300,18 +7304,27 @@ class LibvirtDriver(driver.ComputeDriver):
:returns: a JSON string containing a list of the assignable PCI
devices information
"""
dev_flags = (libvirt.VIR_CONNECT_LIST_NODE_DEVICES_CAP_NET |
libvirt.VIR_CONNECT_LIST_NODE_DEVICES_CAP_PCI_DEV)
dev_flags = (
libvirt.VIR_CONNECT_LIST_NODE_DEVICES_CAP_NET |
libvirt.VIR_CONNECT_LIST_NODE_DEVICES_CAP_PCI_DEV
)
if self._host.has_min_version(
lv_ver=MIN_LIBVIRT_VDPA, hv_ver=MIN_QEMU_VDPA,
):
dev_flags |= libvirt.VIR_CONNECT_LIST_NODE_DEVICES_CAP_VDPA
devices = {
dev.name(): dev for dev in
self._host.list_all_devices(flags=dev_flags)
}
net_devs = [dev for dev in devices.values() if "net" in dev.listCaps()]
vdpa_devs = [
dev for dev in devices.values() if "vdpa" in dev.listCaps()
]
pci_info = [
self._host._get_pcidev_info(name, dev, net_devs)
self._host._get_pcidev_info(name, dev, net_devs, vdpa_devs)
for name, dev in devices.items() if "pci" in dev.listCaps()
]
return jsonutils.dumps(pci_info)
def _get_mdev_capabilities_for_dev(self, devname, types=None):

60
nova/virt/libvirt/host.py
View File

@ -1229,7 +1229,8 @@ class Host(object):
self,
devname: str,
dev: 'libvirt.virNodeDevice',
net_devs: ty.List['libvirt.virNodeDevice']
net_devs: ty.List['libvirt.virNodeDevice'],
vdpa_devs: ty.List['libvirt.virNodeDevice'],
) -> ty.Dict[str, ty.Union[str, dict]]:
"""Returns a dict of PCI device."""
@ -1237,7 +1238,8 @@ class Host(object):
cfgdev: vconfig.LibvirtConfigNodeDevice,
pci_address: str,
device: 'libvirt.virNodeDevice',
net_devs: ty.List['libvirt.virNodeDevice']
net_devs: ty.List['libvirt.virNodeDevice'],
vdpa_devs: ty.List['libvirt.virNodeDevice'],
) -> ty.Dict[str, str]:
"""Get a PCI device's device type.
@ -1246,6 +1248,7 @@ class Host(object):
Function (VF).
"""
net_dev_parents = {dev.parent() for dev in net_devs}
vdpa_parents = {dev.parent() for dev in vdpa_devs}
for fun_cap in cfgdev.pci_capability.fun_capability:
if fun_cap.type == 'virt_functions':
return {
@ -1271,6 +1274,8 @@ class Host(object):
parent_ifname = pci_utils.get_ifname_by_pci_address(
pci_address, pf_interface=True)
result['parent_ifname'] = parent_ifname
if device.name() in vdpa_parents:
result['dev_type'] = fields.PciDeviceType.VDPA
return result
return {'dev_type': fields.PciDeviceType.STANDARD}
@ -1279,18 +1284,20 @@ class Host(object):
device_dict: dict,
device: 'libvirt.virNodeDevice',
net_devs: ty.List['libvirt.virNodeDevice']
) -> ty.Dict[str, ty.Dict[str, ty.Optional[ty.List[str]]]]:
) -> ty.Dict[str, ty.Dict[str, ty.Any]]:
"""Get PCI VF device's additional capabilities.
If a PCI device is a virtual function, this function reads the PCI
parent's network capabilities (must be always a NIC device) and
appends this information to the device's dictionary.
"""
caps: ty.Dict[str, ty.Dict[str, ty.Any]] = {}
if device_dict.get('dev_type') == fields.PciDeviceType.SRIOV_VF:
pcinet_info = self._get_pcinet_info(device, net_devs)
if pcinet_info:
return {'capabilities': {'network': pcinet_info}}
return {}
return caps
xmlstr = dev.XMLDesc(0)
cfgdev = vconfig.LibvirtConfigNodeDevice()
@ -1313,10 +1320,53 @@ class Host(object):
# requirement by DataBase Model
device['label'] = 'label_%(vendor_id)s_%(product_id)s' % device
device.update(_get_device_type(cfgdev, address, dev, net_devs))
device.update(
_get_device_type(cfgdev, address, dev, net_devs, vdpa_devs))
device.update(_get_device_capabilities(device, dev, net_devs))
return device
def get_vdpa_nodedev_by_address(
self, pci_address: str,
) -> vconfig.LibvirtConfigNodeDevice:
"""Finds a vDPA device by the parent VF PCI device address.
:param pci_address: Parent PCI device address
:returns: A libvirt nodedev representing the vDPA device
:raises: StopIteration if not found
"""
dev_flags = (
libvirt.VIR_CONNECT_LIST_NODE_DEVICES_CAP_VDPA |
libvirt.VIR_CONNECT_LIST_NODE_DEVICES_CAP_PCI_DEV
)
devices = {
dev.name(): dev for dev in
self.list_all_devices(flags=dev_flags)}
vdpa_devs = [
dev for dev in devices.values() if "vdpa" in dev.listCaps()]
pci_info = [
self._get_pcidev_info(name, dev, [], vdpa_devs) for name, dev
in devices.items() if "pci" in dev.listCaps()]
parent_dev = next(
dev for dev in pci_info if dev['address'] == pci_address)
vdpa_dev = next(
dev for dev in vdpa_devs if dev.parent() == parent_dev['dev_id'])
xmlstr = vdpa_dev.XMLDesc(0)
cfgdev = vconfig.LibvirtConfigNodeDevice()
cfgdev.parse_str(xmlstr)
return cfgdev
def get_vdpa_device_path(
self, pci_address: str,
) -> str:
"""Finds a vDPA device path by the parent VF PCI device address.
:param pci_address: Parent PCI device address
:returns: Device path as string
:raises: StopIteration if not found
"""
nodedev = self.get_vdpa_nodedev_by_address(pci_address)
return nodedev.vdpa_capability.dev_path
def list_pci_devices(self, flags=0):
"""Lookup pci devices.