starlingx
/
config
Code Issues Proposed changes

Fix N3000 FPGA SR-IOV config for split NICs 

This commit fixes an issue that can occur if a user
creates an SR-IOV interface of type VF with a parent
SR-IOV interface that belongs to a NIC on an Intel
N3000 FPGA.

The FPGA is reset on every worker node bootup (if present),
clearing all SR-IOV config.  Because of this, the current
puppet code waits for the reset to be completed before
restarting the interface via the sysconfig network interface
scripts.  Because the VF interface is a separate hieradata
entry than the parent interface, this can cause a race
condition where the device is re-initialized after one of
the child/parent interfaces has already bound a driver to the
interface.

Since the whole point of the VF interfaces is to 'split'
a NIC to allow multiple SR-IOV VF drivers on one physical NIC,
this commit makes a single hieradata entry for the parent
interface, rather than individual entries for the parent and
child(ren). The information to bind the child interfaces
appropriately is embedded in the vf_config dict of the parent
interface.

Change-Id: Iad34de8ae1b913a1c188e5473e0c92cdf8007ba2
Partial-Bug: #1885229
Signed-off-by: Steven Webster <steven.webster@windriver.com>
This commit is contained in:
Steven Webster 2020-06-26 22:57:35 -04:00
parent 93be0c8d08
commit ff1be48bf0
3 changed files with 146 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
sysinv/sysinv/sysinv/sysinv/puppet/interface.py
View File

@ -1024,17 +1024,13 @@ def get_sriov_interface_vf_addrs(context, iface, vf_addr_list):
return interface.get_sriov_interface_vf_addrs(context, iface, vf_addr_list)
def get_sriov_config(context, iface):
def get_sriov_vf_config(context, iface, port, vf_config):
"""
Returns an SR-IOV interface config dictionary.
Determine the virtual function config for an SR-IOV interface.
"""
vf_driver = iface['sriov_vf_driver']
vf_config = {}
port = interface.get_sriov_interface_port(context, iface)
if not port:
return {}
# Calculate the VF addresses to assign to a logical VF interface,
# taking into account any upper or lower interfaces.
vf_addr_list = ''
vf_addrs = port.get('sriov_vfs_pci_address', None)
if vf_addrs:
@ -1043,23 +1039,21 @@ def get_sriov_config(context, iface):
context, iface, vf_addr_list)
vf_addr_list = ",".join(vf_addr_list)
if vf_driver:
if constants.SRIOV_DRIVER_TYPE_VFIO in vf_driver:
vf_driver = constants.SRIOV_DRIVER_VFIO_PCI
elif constants.SRIOV_DRIVER_TYPE_NETDEVICE in vf_driver:
if port['sriov_vf_driver'] is not None:
vf_driver = port['sriov_vf_driver']
else:
# Should not happen, but in this case the vf driver
# will be determined by the kernel. That is,
# no explicit bind will be performed by Puppet.
vf_driver = None
# Format the vf addresses as quoted strings in order to prevent
# puppet from treating the address as a time/date value
vf_addrs = [quoted_str(addr.strip())
for addr in vf_addr_list.split(",") if addr]
# Get the user specified VF driver, if any. If the driver is
# None, the driver will be determined by the kernel. That is,
# No explicit bind will be done.
vf_driver = iface.get('sriov_vf_driver', None)
if vf_driver:
if constants.SRIOV_DRIVER_TYPE_VFIO in vf_driver:
vf_driver = constants.SRIOV_DRIVER_VFIO_PCI
elif constants.SRIOV_DRIVER_TYPE_NETDEVICE in vf_driver:
vf_driver = port.get('sriov_vf_driver', None)
for addr in vf_addrs:
vf_config.update({
addr: {
@ -1068,6 +1062,26 @@ def get_sriov_config(context, iface):
}
})
if iface.get('used_by', None):
upper_ifaces = iface['used_by']
for upper_ifname in upper_ifaces:
upper_iface = context['interfaces'][upper_ifname]
get_sriov_vf_config(context, upper_iface, port, vf_config)
def get_sriov_config(context, iface):
"""
Returns an SR-IOV interface config dictionary.
"""
vf_config = {}
if iface['iftype'] != constants.INTERFACE_TYPE_ETHERNET:
return {}
port = interface.get_sriov_interface_port(context, iface)
if not port:
return {}
# Include the desired number of VFs if the device supports SR-IOV
# config via sysfs and is not a sub-interface
num_vfs = None
@ -1075,6 +1089,8 @@ def get_sriov_config(context, iface):
and iface['iftype'] != constants.INTERFACE_TYPE_VF):
num_vfs = iface['sriov_numvfs']
get_sriov_vf_config(context, iface, port, vf_config)
config = {
'ifname': iface['ifname'],
'addr': quoted_str(port['pciaddr'].strip()),

1
sysinv/sysinv/sysinv/sysinv/tests/db/utils.py
View File

@ -968,6 +968,7 @@ def get_test_ethernet_port(**kw):
'sriov_numvfs': kw.get('sriov_numvfs'),
'sriov_vf_driver': kw.get('sriov_vf_driver'),
'sriov_vf_pdevice_id': kw.get('sriov_vf_pdevice_id'),
'sriov_vfs_pci_address': kw.get('sriov_vfs_pci_address'),
'driver': kw.get('driver'),
'numa_node': kw.get('numa_node', -1)
}

151
sysinv/sysinv/sysinv/sysinv/tests/puppet/test_interface.py
View File

@ -12,7 +12,6 @@ import yaml
from sysinv.common import constants
from sysinv.puppet import interface
from sysinv.puppet import puppet
from sysinv.puppet import quoted_str
from sysinv.objects import base as objbase
from sysinv.tests.db import base as dbbase
@ -153,7 +152,7 @@ class InterfaceTestCaseMixin(base.PuppetTestCaseMixin):
'networktype': networktype,
'imtu': 1500,
'sriov_numvfs': kwargs.get('sriov_numvfs', 0),
'sriov_vf_driver': kwargs.get('sriov_vf_driver', None)}
'sriov_vf_driver': kwargs.get('iface_sriov_vf_driver', None)}
db_interface = dbutils.create_test_interface(**interface)
self.interfaces.append(db_interface)
@ -170,7 +169,7 @@ class InterfaceTestCaseMixin(base.PuppetTestCaseMixin):
'pciaddr': kwargs.get('pciaddr',
'0000:00:00.' + str(port_id + 1)),
'dev_id': kwargs.get('dev_id', 0),
'sriov_vf_driver': kwargs.get('sriov_vf_driver', None),
'sriov_vf_driver': kwargs.get('port_sriov_vf_driver', None),
'sriov_vf_pdevice_id': kwargs.get('sriov_vf_pdevice_id', None),
'sriov_vfs_pci_address': kwargs.get('sriov_vfs_pci_address', '')}
db_port = dbutils.create_test_ethernet_port(**port)
@ -1123,29 +1122,16 @@ class InterfaceTestCase(InterfaceTestCaseMixin, dbbase.BaseHostTestCase):
def _get_sriov_config(self, ifname='default',
vf_driver=constants.SRIOV_DRIVER_TYPE_VFIO,
vf_addrs=None, num_vfs=2,
pf_addr=None, device_id='1572', port_name="eth0"):
if vf_addrs is None:
vf_addrs = []
num_vfs=2, pf_addr=None, device_id='1572',
port_name="eth0", vf_config=None):
if vf_config is None:
vf_config = {}
config = {'ifname': ifname,
'addr': pf_addr if pf_addr else self.port['pciaddr'],
'device_id': device_id,
'num_vfs': num_vfs,
'port_name': port_name,
'vf_config': {}}
if vf_addrs:
for addr in vf_addrs:
vf_config = config['vf_config']
if addr in vf_config.keys():
vf_config.update({addr: {
'addr': addr,
'driver': vf_driver
}})
else:
vf_config[addr] = {
'addr': addr,
'driver': vf_driver
}
'vf_config': vf_config}
return config
@ -1454,60 +1440,141 @@ class InterfaceTestCase(InterfaceTestCaseMixin, dbbase.BaseHostTestCase):
config = self._create_sriov_vf_config(
constants.SRIOV_DRIVER_TYPE_NETDEVICE, 'i40evf', vf_addr_list,
num_vfs)
expected_vf_config = {
'0000:81:00.0': {'addr': '0000:81:00.0', 'driver': 'i40evf'},
'0000:81:01.0': {'addr': '0000:81:01.0', 'driver': 'i40evf'}
}
expected = self._get_sriov_config(
self.iface['ifname'], 'i40evf',
[quoted_str(vf_addr1),
quoted_str(vf_addr2)],
num_vfs,
ifname=self.iface['ifname'],
vf_driver='i40evf',
num_vfs=num_vfs,
device_id=device_id,
port_name=port_name)
port_name=port_name,
vf_config=expected_vf_config,)
self.assertEqual(expected, config)
def test_get_sriov_config_vfio(self):
vf_addr1 = "0000:81:00.0"
vf_addr2 = "0000:81:01.0"
device_id = '1572'
port_name = 'eth0'
vf_addr_list = "{},{}".format(vf_addr1, vf_addr2)
num_vfs = 4
config = self._create_sriov_vf_config(
constants.SRIOV_DRIVER_TYPE_VFIO, 'i40evf', vf_addr_list,
num_vfs)
expected_vf_config = {
'0000:81:00.0': {'addr': '0000:81:00.0', 'driver': 'vfio-pci'},
'0000:81:01.0': {'addr': '0000:81:01.0', 'driver': 'vfio-pci'}
}
expected = self._get_sriov_config(
self.iface['ifname'], 'vfio-pci',
[quoted_str(vf_addr1),
quoted_str(vf_addr2)],
num_vfs)
ifname=self.iface['ifname'],
vf_driver='vfio-pci',
num_vfs=num_vfs,
device_id=device_id,
port_name=port_name,
vf_config=expected_vf_config)
self.assertEqual(expected, config)
def test_get_sriov_config_default(self):
vf_addr1 = "0000:81:00.0"
vf_addr2 = "0000:81:01.0"
device_id = '1572'
port_name = 'eth0'
vf_addr_list = "{},{}".format(vf_addr1, vf_addr2)
num_vfs = 1
config = self._create_sriov_vf_config(
None, 'i40evf', vf_addr_list, num_vfs)
expected_vf_config = {
'0000:81:00.0': {'addr': '0000:81:00.0', 'driver': None},
'0000:81:01.0': {'addr': '0000:81:01.0', 'driver': None}
}
expected = self._get_sriov_config(
self.iface['ifname'], None,
[quoted_str(vf_addr1),
quoted_str(vf_addr2)],
num_vfs)
ifname=self.iface['ifname'],
vf_driver=None,
device_id=device_id,
port_name=port_name,
num_vfs=num_vfs,
vf_config=expected_vf_config)
self.assertEqual(expected, config)
def test_get_sriov_config_iftype_vf(self):
port, iface = self._create_ethernet_test(
'sriov1', constants.INTERFACE_CLASS_PCI_SRIOV,
constants.NETWORK_TYPE_PCI_SRIOV, sriov_numvfs=2,
sriov_vf_driver=None)
vf = self._create_vf_test("vf1", 1, None, lower_iface=iface)
constants.NETWORK_TYPE_PCI_SRIOV, sriov_numvfs=4,
iface_sriov_vf_driver=None,
port_sriov_vf_driver="iavf",
sriov_vfs_pci_address="0000:b1:02.0,0000:b1:02.1,0000:b1:02.2,0000:b1:02.3")
self._create_vf_test("vf1", 1, 'vfio', lower_iface=iface)
self._update_context()
config = interface.get_sriov_config(self.context, vf)
config = interface.get_sriov_config(self.context, iface)
expected_vf_config = {
'0000:b1:02.0': {'addr': '0000:b1:02.0', 'driver': None},
'0000:b1:02.1': {'addr': '0000:b1:02.1', 'driver': None},
'0000:b1:02.2': {'addr': '0000:b1:02.2', 'driver': None},
'0000:b1:02.3': {'addr': '0000:b1:02.3', 'driver': 'vfio-pci'}
}
expected = self._get_sriov_config(
vf['ifname'], None,
None,
None, pf_addr=port['pciaddr'],
port_name="eth1")
iface['ifname'], None,
num_vfs=4, pf_addr=port['pciaddr'],
port_name="eth1",
vf_config=expected_vf_config)
self.assertEqual(expected, config)
def test_get_sriov_config_iftype_vf_nested(self):
port, iface = self._create_ethernet_test(
'sriov1', constants.INTERFACE_CLASS_PCI_SRIOV,
constants.NETWORK_TYPE_PCI_SRIOV, sriov_numvfs=4,
iface_sriov_vf_driver=None,
port_sriov_vf_driver="iavf",
sriov_vfs_pci_address="0000:b1:02.0,0000:b1:02.1,0000:b1:02.2,0000:b1:02.3")
vf1 = self._create_vf_test("vf1", 2, 'vfio', lower_iface=iface)
self._create_vf_test("vf2", 1, 'netdevice', lower_iface=vf1)
self._update_context()
config = interface.get_sriov_config(self.context, iface)
expected_vf_config = {
'0000:b1:02.0': {'addr': '0000:b1:02.0', 'driver': None},
'0000:b1:02.1': {'addr': '0000:b1:02.1', 'driver': None},
'0000:b1:02.2': {'addr': '0000:b1:02.2', 'driver': 'vfio-pci'},
'0000:b1:02.3': {'addr': '0000:b1:02.3', 'driver': 'iavf'}
}
expected = self._get_sriov_config(
iface['ifname'], None,
num_vfs=4, pf_addr=port['pciaddr'],
port_name="eth1",
vf_config=expected_vf_config)
self.assertEqual(expected, config)
def test_get_sriov_config_iftype_vf_sibling(self):
port, iface = self._create_ethernet_test(
'sriov1', constants.INTERFACE_CLASS_PCI_SRIOV,
constants.NETWORK_TYPE_PCI_SRIOV, sriov_numvfs=4,
iface_sriov_vf_driver=None,
port_sriov_vf_driver="iavf",
sriov_vfs_pci_address="0000:b1:02.0,0000:b1:02.1,0000:b1:02.2,0000:b1:02.3")
self._create_vf_test("vf1", 2, 'vfio', lower_iface=iface)
self._create_vf_test("vf2", 1, 'netdevice', lower_iface=iface)
self._update_context()
config = interface.get_sriov_config(self.context, iface)
expected_vf_config = {
'0000:b1:02.0': {'addr': '0000:b1:02.0', 'driver': None},
'0000:b1:02.1': {'addr': '0000:b1:02.1', 'driver': 'iavf'},
'0000:b1:02.2': {'addr': '0000:b1:02.2', 'driver': 'vfio-pci'},
'0000:b1:02.3': {'addr': '0000:b1:02.3', 'driver': 'vfio-pci'}
}
expected = self._get_sriov_config(
iface['ifname'], None,
num_vfs=4, pf_addr=port['pciaddr'],
port_name="eth1",
vf_config=expected_vf_config)
self.assertEqual(expected, config)
def test_is_a_mellanox_cx3_device_false(self):