SRIOV agent: wait VFs initialization on embedded switch create

Check for configured and actual number of VFs to prevent
device registaration with 0 VFs.

Closes-Bug: #1831622
Change-Id: Ie699d245f8ae2fc1d16b96432d2962788d9dba57
(cherry picked from commit c148c6df46)

neutron/plugins/ml2/drivers/mech_sriov/agent/eswitch_manager.py

@@ -33,6 +33,7 @@ class PciOsWrapper(object):
 
     DEVICE_PATH = "/sys/class/net/%s/device"
     PCI_PATH = "/sys/class/net/%s/device/virtfn%s/net"
+    NUMVFS_PATH = "/sys/class/net/%s/device/sriov_numvfs"
     VIRTFN_FORMAT = r"^virtfn(?P<vf_index>\d+)"
     VIRTFN_REG_EX = re.compile(VIRTFN_FORMAT)
 
@@ -102,6 +103,25 @@ class PciOsWrapper(object):
                 return True
         return False
 
+    @classmethod
+    def get_numvfs(cls, dev_name):
+        """Get configured number of VFs on device
+
+        @param dev_name: pf network device name
+        @return: integer number of VFs or -1
+        if sriov_numvfs file not found (device doesn't support this config)
+        """
+        try:
+            with open(cls.NUMVFS_PATH % dev_name) as f:
+                numvfs = int(f.read())
+                LOG.debug("Number of VFs configured on device %s: %s",
+                    dev_name, numvfs)
+                return numvfs
+        except IOError:
+            LOG.warning("Error reading sriov_numvfs file for device %s, "
+                        "probably not supported by this device", dev_name)
+            return -1
+
 
 class EmbSwitch(object):
     """Class to manage logical embedded switch entity.
@@ -122,6 +142,7 @@ class EmbSwitch(object):
         """
         self.dev_name = dev_name
         self.pci_slot_map = {}
+        self.scanned_pci_list = []
         self.pci_dev_wrapper = pci_lib.PciDeviceIPWrapper(dev_name)
 
         self._load_devices(exclude_devices)
@@ -131,8 +152,8 @@ class EmbSwitch(object):
 
         @param exclude_devices: excluded devices mapping device_name: pci slots
         """
-        scanned_pci_list = PciOsWrapper.scan_vf_devices(self.dev_name)
-        for pci_slot, vf_index in scanned_pci_list:
+        self.scanned_pci_list = PciOsWrapper.scan_vf_devices(self.dev_name)
+        for pci_slot, vf_index in self.scanned_pci_list:
             if pci_slot not in exclude_devices:
                 self.pci_slot_map[pci_slot] = vf_index
 
@@ -258,6 +279,7 @@ class ESwitchManager(object):
             cls._instance = super(ESwitchManager, cls).__new__(cls)
             cls.emb_switches_map = {}
             cls.pci_slot_map = {}
+            cls.skipped_devices = set()
         return cls._instance
 
     def device_exists(self, device_mac, pci_slot):
@@ -400,9 +422,33 @@ class ESwitchManager(object):
 
     def _create_emb_switch(self, phys_net, dev_name, exclude_devices):
         embedded_switch = EmbSwitch(dev_name, exclude_devices)
+        numvfs = PciOsWrapper.get_numvfs(dev_name)
+        if numvfs == 0:
+            # numvfs might be 0 on pre-up state of a device
+            # giving such devices one more chance to initialize
+            if dev_name not in self.skipped_devices:
+                self.skipped_devices.add(dev_name)
+                LOG.info("Device %s has 0 VFs configured. Skipping "
+                         "for now to let the device initialize", dev_name)
+                return
+            else:
+                # looks like device indeed has 0 VFs configured
+                # it is probably used just as direct-physical
+                LOG.info("Device %s has 0 VFs configured", dev_name)
+
+        numvfs_cur = len(embedded_switch.scanned_pci_list)
+        if numvfs >= 0 and numvfs > numvfs_cur:
+            LOG.info("Not all VFs were initialized on device %(device)s: "
+                     "expected - %(expected)s, actual - %(actual)s. Skipping.",
+                     {'device': dev_name, 'expected': numvfs,
+                      'actual': numvfs_cur})
+            self.skipped_devices.add(dev_name)
+            return
+
         self.emb_switches_map.setdefault(phys_net, []).append(embedded_switch)
         for pci_slot in embedded_switch.get_pci_slot_list():
             self.pci_slot_map[pci_slot] = embedded_switch
+        self.skipped_devices.discard(dev_name)
 
     def _get_emb_eswitch(self, device_mac, pci_slot):
         """Get embedded switch.

neutron/tests/unit/plugins/ml2/drivers/mech_sriov/agent/test_eswitch_manager.py

@@ -379,6 +379,48 @@ class TestESwitchManagerApi(base.BaseTestCase):
         self._test_clear_rate(self.MIN_RATE, self.WRONG_PCI, passed=False,
                               mac_address={})
 
+    def test_create_emb_switch(self):
+        DEVICES = [('0000:04:00.1', 0),
+                   ('0000:04:00.2', 1)]
+        with mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent."
+                        "eswitch_manager.PciOsWrapper.scan_vf_devices",
+                        side_effect=[[], DEVICES]), \
+                mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent."
+                           "eswitch_manager.PciOsWrapper.get_numvfs",
+                           return_value=2):
+            physnet = 'test_create_emb_switch'
+            self.assertNotIn(physnet, self.eswitch_mgr.emb_switches_map)
+            # first time device will not be added as no VFs returned
+            self.eswitch_mgr._create_emb_switch(physnet, 'dev1', [])
+            self.assertNotIn(physnet, self.eswitch_mgr.emb_switches_map)
+            self.assertEqual({'dev1'}, self.eswitch_mgr.skipped_devices)
+
+            # second time device should be added with 2 VFs
+            self.eswitch_mgr._create_emb_switch(physnet, 'dev1', [])
+            self.assertIn(physnet, self.eswitch_mgr.emb_switches_map)
+            self.assertEqual(set(), self.eswitch_mgr.skipped_devices)
+            self.assertIn('0000:04:00.1', self.eswitch_mgr.pci_slot_map)
+            self.assertIn('0000:04:00.2', self.eswitch_mgr.pci_slot_map)
+
+    def test_create_emb_switch_zero_vfs(self):
+        with mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent."
+                        "eswitch_manager.PciOsWrapper.scan_vf_devices",
+                        return_value=[]), \
+                mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent."
+                           "eswitch_manager.PciOsWrapper.get_numvfs",
+                           return_value=0):
+            physnet = 'test_create_emb_switch'
+            self.assertNotIn(physnet, self.eswitch_mgr.emb_switches_map)
+            # first time device will not be added
+            self.eswitch_mgr._create_emb_switch(physnet, 'dev1', [])
+            self.assertNotIn(physnet, self.eswitch_mgr.emb_switches_map)
+            self.assertEqual({'dev1'}, self.eswitch_mgr.skipped_devices)
+
+            # second time device should be added with 0 VFs
+            self.eswitch_mgr._create_emb_switch(physnet, 'dev1', [])
+            self.assertIn(physnet, self.eswitch_mgr.emb_switches_map)
+            self.assertEqual(set(), self.eswitch_mgr.skipped_devices)
+
 
 class TestEmbSwitch(base.BaseTestCase):
     DEV_NAME = "eth2"
@@ -693,3 +735,17 @@ class TestPciOsWrapper(base.BaseTestCase):
     def test_pf_device_exists_with_dir(self):
         with mock.patch("os.path.isdir", return_value=True):
             self.assertTrue(esm.PciOsWrapper.pf_device_exists('p6p1'))
+
+    def test_get_numvfs(self):
+        with mock.patch("six.moves.builtins.open",
+                        mock.mock_open(read_data="63")) as mock_open:
+            self.assertEqual(63, esm.PciOsWrapper.get_numvfs('dev1'))
+            mock_open.assert_called_once_with(
+                esm.PciOsWrapper.NUMVFS_PATH % 'dev1')
+
+    def test_get_numvfs_no_file(self):
+        with mock.patch("six.moves.builtins.open",
+                        side_effect=IOError()) as mock_open:
+            self.assertEqual(-1, esm.PciOsWrapper.get_numvfs('dev1'))
+            mock_open.assert_called_once_with(
+                esm.PciOsWrapper.NUMVFS_PATH % 'dev1')