SRIOV agent: wait VFs initialization on embedded switch create

Check for configured and actual number of VFs to prevent
device registaration with 0 VFs.

Closes-Bug: #1831622
Change-Id: Ie699d245f8ae2fc1d16b96432d2962788d9dba57
(cherry picked from commit c148c6df46)
This commit is contained in:
Oleg Bondarev 2019-06-04 12:57:48 +04:00
parent 489a841bc5
commit a48f157db5
2 changed files with 104 additions and 2 deletions

View File

@ -33,6 +33,7 @@ class PciOsWrapper(object):
DEVICE_PATH = "/sys/class/net/%s/device"
PCI_PATH = "/sys/class/net/%s/device/virtfn%s/net"
NUMVFS_PATH = "/sys/class/net/%s/device/sriov_numvfs"
VIRTFN_FORMAT = r"^virtfn(?P<vf_index>\d+)"
VIRTFN_REG_EX = re.compile(VIRTFN_FORMAT)
@ -102,6 +103,25 @@ class PciOsWrapper(object):
return True
return False
@classmethod
def get_numvfs(cls, dev_name):
"""Get configured number of VFs on device
@param dev_name: pf network device name
@return: integer number of VFs or -1
if sriov_numvfs file not found (device doesn't support this config)
"""
try:
with open(cls.NUMVFS_PATH % dev_name) as f:
numvfs = int(f.read())
LOG.debug("Number of VFs configured on device %s: %s",
dev_name, numvfs)
return numvfs
except IOError:
LOG.warning("Error reading sriov_numvfs file for device %s, "
"probably not supported by this device", dev_name)
return -1
class EmbSwitch(object):
"""Class to manage logical embedded switch entity.
@ -122,6 +142,7 @@ class EmbSwitch(object):
"""
self.dev_name = dev_name
self.pci_slot_map = {}
self.scanned_pci_list = []
self.pci_dev_wrapper = pci_lib.PciDeviceIPWrapper(dev_name)
self._load_devices(exclude_devices)
@ -131,8 +152,8 @@ class EmbSwitch(object):
@param exclude_devices: excluded devices mapping device_name: pci slots
"""
scanned_pci_list = PciOsWrapper.scan_vf_devices(self.dev_name)
for pci_slot, vf_index in scanned_pci_list:
self.scanned_pci_list = PciOsWrapper.scan_vf_devices(self.dev_name)
for pci_slot, vf_index in self.scanned_pci_list:
if pci_slot not in exclude_devices:
self.pci_slot_map[pci_slot] = vf_index
@ -258,6 +279,7 @@ class ESwitchManager(object):
cls._instance = super(ESwitchManager, cls).__new__(cls)
cls.emb_switches_map = {}
cls.pci_slot_map = {}
cls.skipped_devices = set()
return cls._instance
def device_exists(self, device_mac, pci_slot):
@ -400,9 +422,33 @@ class ESwitchManager(object):
def _create_emb_switch(self, phys_net, dev_name, exclude_devices):
embedded_switch = EmbSwitch(dev_name, exclude_devices)
numvfs = PciOsWrapper.get_numvfs(dev_name)
if numvfs == 0:
# numvfs might be 0 on pre-up state of a device
# giving such devices one more chance to initialize
if dev_name not in self.skipped_devices:
self.skipped_devices.add(dev_name)
LOG.info("Device %s has 0 VFs configured. Skipping "
"for now to let the device initialize", dev_name)
return
else:
# looks like device indeed has 0 VFs configured
# it is probably used just as direct-physical
LOG.info("Device %s has 0 VFs configured", dev_name)
numvfs_cur = len(embedded_switch.scanned_pci_list)
if numvfs >= 0 and numvfs > numvfs_cur:
LOG.info("Not all VFs were initialized on device %(device)s: "
"expected - %(expected)s, actual - %(actual)s. Skipping.",
{'device': dev_name, 'expected': numvfs,
'actual': numvfs_cur})
self.skipped_devices.add(dev_name)
return
self.emb_switches_map.setdefault(phys_net, []).append(embedded_switch)
for pci_slot in embedded_switch.get_pci_slot_list():
self.pci_slot_map[pci_slot] = embedded_switch
self.skipped_devices.discard(dev_name)
def _get_emb_eswitch(self, device_mac, pci_slot):
"""Get embedded switch.

View File

@ -379,6 +379,48 @@ class TestESwitchManagerApi(base.BaseTestCase):
self._test_clear_rate(self.MIN_RATE, self.WRONG_PCI, passed=False,
mac_address={})
def test_create_emb_switch(self):
DEVICES = [('0000:04:00.1', 0),
('0000:04:00.2', 1)]
with mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent."
"eswitch_manager.PciOsWrapper.scan_vf_devices",
side_effect=[[], DEVICES]), \
mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent."
"eswitch_manager.PciOsWrapper.get_numvfs",
return_value=2):
physnet = 'test_create_emb_switch'
self.assertNotIn(physnet, self.eswitch_mgr.emb_switches_map)
# first time device will not be added as no VFs returned
self.eswitch_mgr._create_emb_switch(physnet, 'dev1', [])
self.assertNotIn(physnet, self.eswitch_mgr.emb_switches_map)
self.assertEqual({'dev1'}, self.eswitch_mgr.skipped_devices)
# second time device should be added with 2 VFs
self.eswitch_mgr._create_emb_switch(physnet, 'dev1', [])
self.assertIn(physnet, self.eswitch_mgr.emb_switches_map)
self.assertEqual(set(), self.eswitch_mgr.skipped_devices)
self.assertIn('0000:04:00.1', self.eswitch_mgr.pci_slot_map)
self.assertIn('0000:04:00.2', self.eswitch_mgr.pci_slot_map)
def test_create_emb_switch_zero_vfs(self):
with mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent."
"eswitch_manager.PciOsWrapper.scan_vf_devices",
return_value=[]), \
mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent."
"eswitch_manager.PciOsWrapper.get_numvfs",
return_value=0):
physnet = 'test_create_emb_switch'
self.assertNotIn(physnet, self.eswitch_mgr.emb_switches_map)
# first time device will not be added
self.eswitch_mgr._create_emb_switch(physnet, 'dev1', [])
self.assertNotIn(physnet, self.eswitch_mgr.emb_switches_map)
self.assertEqual({'dev1'}, self.eswitch_mgr.skipped_devices)
# second time device should be added with 0 VFs
self.eswitch_mgr._create_emb_switch(physnet, 'dev1', [])
self.assertIn(physnet, self.eswitch_mgr.emb_switches_map)
self.assertEqual(set(), self.eswitch_mgr.skipped_devices)
class TestEmbSwitch(base.BaseTestCase):
DEV_NAME = "eth2"
@ -693,3 +735,17 @@ class TestPciOsWrapper(base.BaseTestCase):
def test_pf_device_exists_with_dir(self):
with mock.patch("os.path.isdir", return_value=True):
self.assertTrue(esm.PciOsWrapper.pf_device_exists('p6p1'))
def test_get_numvfs(self):
with mock.patch("six.moves.builtins.open",
mock.mock_open(read_data="63")) as mock_open:
self.assertEqual(63, esm.PciOsWrapper.get_numvfs('dev1'))
mock_open.assert_called_once_with(
esm.PciOsWrapper.NUMVFS_PATH % 'dev1')
def test_get_numvfs_no_file(self):
with mock.patch("six.moves.builtins.open",
side_effect=IOError()) as mock_open:
self.assertEqual(-1, esm.PciOsWrapper.get_numvfs('dev1'))
mock_open.assert_called_once_with(
esm.PciOsWrapper.NUMVFS_PATH % 'dev1')