Configure switchdev mode in SR-IOV in os-net-config

While sriov vfs are created now using os-net-config,
so doing some modification to configure switchdev mode there

Closes-Bug: #1818034
(cherry picked from commit 3dcad07311)
Conflicts:
	os_net_config/objects.py
	os_net_config/schema.yaml

Change-Id: I7c85ab380dc4f0377a65fc954fcce3e3d39f5029
Signed-off-by: waleed mousa <waleedm@mellanox.com>

etc/os-net-config/samples/sriov_pf_switchdev.json

@@ -0,0 +1,84 @@
+{
+    "network_config": [
+        {
+            "type": "sriov_pf",
+            "name": "p2p1",
+            "numvfs": 10,
+            "use_dhcp": false,
+            "promisc": true,
+            "link_mode": "switchdev"
+        },
+        {
+            "type": "ovs_bridge",
+            "name": "br-pfs",
+            "members": [
+                {
+                    "type": "sriov_pf",
+                    "name": "p2p2",
+                    "numvfs": 10,
+                    "promisc": true,
+                    "use_dhcp": false,
+                    "link_mode": "switchdev"
+                }
+            ],
+            "use_dhcp": true
+        },
+        {
+            "type": "ovs_bridge",
+            "name": "br-bond",
+            "use_dhcp": true,
+            "members": [
+                {
+                    "type": "ovs_bond",
+                    "name": "bond_pf",
+                    "ovs_options": "bond_mode=active-backup",
+                    "members": [
+                        {
+                            "type": "sriov_pf",
+                            "name": "p5p1",
+                            "numvfs": 10,
+                            "primary": true,
+                            "promisc": true,
+                            "use_dhcp": false,
+                            "link_mode": "switchdev"
+                        },
+                        {
+                            "type": "sriov_pf",
+                            "name": "p5p2",
+                            "numvfs": 10,
+                            "promisc": true,
+                            "use_dhcp": false,
+                            "link_mode": "switchdev"
+                        }
+                    ]
+                }
+            ]
+        },
+        {
+            "type": "linux_bond",
+            "name": "bond_lnx",
+            "use_dhcp": true,
+            "bonding_options": "mode=active-backup",
+            "members": [
+                {
+                    "type": "sriov_pf",
+                    "name": "p6p1",
+                    "numvfs": 10,
+                    "primary": true,
+                    "promisc": true,
+                    "use_dhcp": false,
+                    "link_mode": "switchdev"
+                },
+                {
+                    "type": "sriov_pf",
+                    "name": "p6p2",
+                    "numvfs": 10,
+                    "promisc": true,
+                    "use_dhcp": false,
+                    "link_mode": "switchdev"
+
+                }
+            ]
+        }
+    ]
+}

etc/os-net-config/samples/sriov_pf_switchdev.yaml

@@ -0,0 +1,87 @@
+network_config:
+  # sriov_pf type shall be used to configure the PF's of NICs.
+  # The numvfs configured for the PF's shall be set on the sriov_numvfs of the
+  # sysfs for the corresponding NIC and the persistence of the same across reboots
+  # shall be handled.
+  # The link_mode configured the mode of the sriov_pf which can be "switchdev"
+  # or legacy.
+  # In case fo switchdev, you will enable the hardware offloading and VF-LAG
+  # capabilities
+  # For now, if you configured link_mode with "switchdev" you will not be able
+  # to use any of its sriov_vfs, the vfs will be only used for vms.
+  -
+    type: sriov_pf
+    # nic name or nic number of the NIC that needs to be configured for SRIOV
+    name: p2p1
+    # number of VFs required on the particular NIC
+    numvfs: 10
+    addresses:
+    - ip_netmask: 192.0.2.1/24
+    # Allow all the traffic received. Default: true
+    promisc: true
+    # The mode of sriov_pf which:
+    # - switchdev
+    # - legacy (default)
+    link_mode: switchdev
+
+  -
+    type: ovs_bridge
+    name: br-pfs
+    use_dhcp: true
+    members:
+    -
+      type: sriov_pf
+      name: p2p2
+      numvfs: 10
+      promisc: true
+      use_dhcp: false
+      link_mode: switchdev
+
+  -
+    type: ovs_bridge
+    name: br-bond
+    use_dhcp: true
+    members:
+    -
+      type: ovs_bond
+      name: bond_pf
+      ovs_options: "bond_mode=active-backup"
+      members:
+      -
+        type: sriov_pf
+        name: p5p1
+        numvfs: 10
+        primary: true
+        promisc: true
+        use_dhcp: false
+        link_mode: switchdev
+      -
+        type: sriov_pf
+        name: p5p2
+        numvfs: 10
+        promisc: true
+        use_dhcp: false
+        link_mode: switchdev
+
+  -
+    # VF-LAG capability, which means that the vfs will be bonded in hardware
+    type: linux_bond
+    name: bond_lnx
+    use_dhcp: true
+    bonding_options: "mode=active-backup"
+    members:
+    -
+      type: sriov_pf
+      name: p6p1
+      numvfs: 10
+      primary: true
+      promisc: true
+      use_dhcp: false
+      link_mode: switchdev
+    -
+      type: sriov_pf
+      name: p6p2
+      numvfs: 10
+      promisc: true
+      use_dhcp: false
+      link_mode: switchdev

os_net_config/impl_ifcfg.py

@@ -857,7 +857,8 @@ class IfcfgNetConfig(os_net_config.NetConfig):
         data = self._add_common(sriov_pf)
         logger.debug('sriov pf data: %s' % data)
         utils.update_sriov_pf_map(sriov_pf.name, sriov_pf.numvfs,
-                                  self.noop, promisc=sriov_pf.promisc)
+                                  self.noop, promisc=sriov_pf.promisc,
+                                  link_mode=sriov_pf.link_mode)
         self.interface_data[sriov_pf.name] = data
 
     def add_sriov_vf(self, sriov_vf):

os_net_config/objects.py

@@ -1317,7 +1317,7 @@ class SriovPF(_BaseOpts):
                  addresses=None, routes=None, mtu=None, primary=False,
                  nic_mapping=None, persist_mapping=False, defroute=True,
                  dhclient_args=None, dns_servers=None, nm_controlled=False,
-                 onboot=True, members=None, promisc=None):
+                 onboot=True, members=None, promisc=None, link_mode='legacy'):
         addresses = addresses or []
         routes = routes or []
         dns_servers = dns_servers or []
@@ -1333,6 +1333,7 @@ class SriovPF(_BaseOpts):
         else:
             self.name = name
         self.promisc = promisc
+        self.link_mode = link_mode
 
     @staticmethod
     def get_on_off(config):
@@ -1350,8 +1351,13 @@ class SriovPF(_BaseOpts):
         # SR-IOV PF - promisc: on (default)
         promisc = json.get('promisc', True)
         promisc = SriovPF.get_on_off(promisc)
+        link_mode = json.get('link_mode', 'legacy')
+        if link_mode not in ['legacy', 'switchdev']:
+            msg = 'Expecting link_mode to match legacy/switchdev'
+            raise InvalidConfigException(msg)
         opts = _BaseOpts.base_opts_from_json(json)
-        return SriovPF(name, numvfs, *opts, promisc=promisc)
+        return SriovPF(name, numvfs, *opts, promisc=promisc,
+                       link_mode=link_mode)
 
 
 class OvsDpdkBond(_BaseOpts):

os_net_config/schema.yaml

@@ -33,6 +33,11 @@ definitions:
     sriov_vf_state_string:
         type: string
         pattern: "^(auto|enable|disable)$"
+    sriov_link_mode_or_param:
+        oneOf:
+          - type: string
+            pattern: "^(legacy|switchdev)$"
+          - $ref: "#/definitions/param"
     # MAC address type
     mac_address_string:
         type: string
@@ -291,6 +296,8 @@ definitions:
                 $ref: "#/definitions/bool_or_param"
             onboot:
                 $ref: "#/definitions/bool_or_param"
+            link_mode:
+                $ref: "#/definitions/sriov_link_mode_or_param"
         required:
           - type
           - name

os_net_config/sriov_config.py

@@ -25,6 +25,7 @@ import argparse
 import logging
 import os
 import pyudev
+import re
 from six.moves import queue as Queue
 import sys
 import yaml
@@ -32,6 +33,7 @@ import yaml
 from oslo_concurrency import processutils
 
 logger = logging.getLogger(__name__)
+_SYS_CLASS_NET = '/sys/class/net'
 # Create a queue for passing the udev network events
 vf_queue = Queue.Queue()
 
@@ -94,6 +96,8 @@ def configure_sriov_pf():
     observer.start()
 
     sriov_map = _get_sriov_map()
+    MLNX_UNBIND_FILE_PATH = "/sys/bus/pci/drivers/mlx5_core/unbind"
+    MLNX_VENDOR_ID = "0x15b3"
     for item in sriov_map:
         if item['device_type'] == 'pf':
             _pf_interface_up(item)
@@ -108,6 +112,19 @@ def configure_sriov_pf():
                 raise SRIOVNumvfsException(msg)
             # Wait for the creation of VFs for each PF
             _wait_for_vf_creation(item['name'], item['numvfs'])
+            # Configure switchdev mode
+            vendor_id = get_vendor_id(item['name'])
+            if (item.get('link_mode') == "switchdev" and
+                    vendor_id == MLNX_VENDOR_ID):
+                vf_pcis_list = get_vf_pcis_list(item['name'])
+                for vf_pci in vf_pcis_list:
+                    vf_pci_path = "/sys/bus/pci/devices/%s/driver" % vf_pci
+                    if os.path.exists(vf_pci_path):
+                        with open(MLNX_UNBIND_FILE_PATH, 'w') as f:
+                            f.write("%s" % vf_pci)
+                configure_switchdev(item['name'])
+                if_up_interface(item['name'])
+
     observer.stop()
 
 
@@ -139,6 +156,30 @@ def _wait_for_vf_creation(pf_name, numvfs):
     logger.info("Required VFs are created for PF %s" % pf_name)
 
 
+def configure_switchdev(pf_name):
+    pf_pci = get_pf_pci(pf_name)
+    pf_device_id = get_pf_device_id(pf_name)
+    if pf_device_id == "0x1013" or pf_device_id == "0x1015":
+        try:
+            processutils.execute('/usr/sbin/devlink', 'dev', 'eswitch', 'set',
+                                 'pci/%s' % pf_pci, 'inline-mode', 'transport')
+        except processutils.ProcessExecutionError:
+            logger.error("Failed to set inline-mode to transport")
+            raise
+    try:
+        processutils.execute('/usr/sbin/devlink', 'dev', 'eswitch', 'set',
+                             'pci/%s' % pf_pci, 'mode', 'switchdev')
+    except processutils.ProcessExecutionError:
+        logger.error("Failed to set mode to switchdev")
+        raise
+    try:
+        processutils.execute('/usr/sbin/ethtool', '-K', pf_name,
+                             'hw-tc-offload', 'on')
+    except processutils.ProcessExecutionError:
+        logger.error("Failed to enable hw-tc-offload")
+        raise
+
+
 def run_ip_config_cmd(*cmd, **kwargs):
     logger.info("Running %s" % ' '.join(cmd))
     try:
@@ -156,6 +197,51 @@ def _pf_interface_up(pf_device):
     run_ip_config_cmd('ip', 'link', 'set', 'dev', pf_device['name'], 'up')
 
 
+def get_vendor_id(ifname):
+    try:
+        with open(os.path.join(_SYS_CLASS_NET, ifname, "device/vendor"),
+                  'r') as f:
+            out = f.read().strip()
+        return out
+    except IOError:
+        return
+
+
+def get_pf_pci(pf_name):
+    pf_pci_path = os.path.join(_SYS_CLASS_NET, pf_name, "device/uevent")
+    pf_info = get_file_data(pf_pci_path)
+    pf_pci = re.search(r'PCI_SLOT_NAME=(.*)', pf_info, re.MULTILINE).group(1)
+    return pf_pci
+
+
+def get_pf_device_id(pf_name):
+    pf_device_path = os.path.join(_SYS_CLASS_NET, pf_name, "device/device")
+    pf_device_id = get_file_data(pf_device_path).strip()
+    return pf_device_id
+
+
+def get_vf_pcis_list(pf_name):
+    vf_pcis_list = []
+    listOfPfFiles = os.listdir(os.path.join(_SYS_CLASS_NET, pf_name,
+                                            "device"))
+    for pf_file in listOfPfFiles:
+        if pf_file.startswith("virtfn"):
+            vf_info = get_file_data(os.path.join(_SYS_CLASS_NET, pf_name,
+                                    "device", pf_file, "uevent"))
+            vf_pcis_list.append(re.search(r'PCI_SLOT_NAME=(.*)',
+                                          vf_info, re.MULTILINE).group(1))
+    return vf_pcis_list
+
+
+def if_up_interface(device):
+    logger.info("Running /sbin/ifup %s" % device)
+    try:
+        processutils.execute('/sbin/ifup', device)
+    except processutils.ProcessExecutionError:
+        logger.error("Failed to ifup  %s" % device)
+        raise
+
+
 def configure_sriov_vf():
     sriov_map = _get_sriov_map()
     for item in sriov_map:

os_net_config/tests/test_impl_ifcfg.py

@@ -1309,10 +1309,12 @@ NETMASK=255.255.255.0
 
         pf = objects.SriovPF(name='nic3', numvfs=10)
 
-        def test_update_sriov_pf_map(name, numvfs, noop, promisc=None):
+        def test_update_sriov_pf_map(name, numvfs, noop, promisc=None,
+                                     link_mode='legacy'):
             self.assertEqual(name, 'eth2')
             self.assertEqual(numvfs, 10)
             self.assertEqual(promisc, None)
+            self.assertEqual(link_mode, 'legacy')
         self.stub_out('os_net_config.utils.update_sriov_pf_map',
                       test_update_sriov_pf_map)
         self.provider.add_sriov_pf(pf)
@@ -1332,10 +1334,12 @@ BOOTPROTO=none
 
         pf = objects.SriovPF(name='nic3', numvfs=10, promisc=True)
 
-        def test_update_sriov_pf_map(name, numvfs, noop, promisc=None):
+        def test_update_sriov_pf_map(name, numvfs, noop, promisc=None,
+                                     link_mode='legacy'):
             self.assertEqual(name, 'eth2')
             self.assertEqual(numvfs, 10)
             self.assertTrue(promisc)
+            self.assertEqual(link_mode, 'legacy')
         self.stub_out('os_net_config.utils.update_sriov_pf_map',
                       test_update_sriov_pf_map)
         self.provider.add_sriov_pf(pf)
@@ -1355,10 +1359,12 @@ BOOTPROTO=none
 
         pf = objects.SriovPF(name='nic3', numvfs=10, promisc=False)
 
-        def test_update_sriov_pf_map(name, numvfs, noop, promisc=None):
+        def test_update_sriov_pf_map(name, numvfs, noop, promisc=None,
+                                     link_mode='legacy'):
             self.assertEqual(name, 'eth2')
             self.assertEqual(numvfs, 10)
             self.assertFalse(promisc)
+            self.assertEqual(link_mode, 'legacy')
         self.stub_out('os_net_config.utils.update_sriov_pf_map',
                       test_update_sriov_pf_map)
         self.provider.add_sriov_pf(pf)

os_net_config/tests/test_objects.py

@@ -1675,6 +1675,7 @@ class TestSriovPF(base.TestCase):
         self.assertEqual(16, pf.numvfs)
         self.assertEqual("off", pf.promisc)
         self.assertFalse(pf.use_dhcp)
+        self.assertEqual("legacy", pf.link_mode)
 
     def test_from_json_numvfs_nic1(self):
         def dummy_mapped_nics(nic_mapping=None):
@@ -1702,6 +1703,27 @@ class TestSriovPF(base.TestCase):
         self.assertFalse(pf.use_dhcp)
         self.assertEqual('on', pf.promisc)
 
+    def test_from_json_link_mode(self):
+        data = '{"type": "sriov_pf", "name": "p6p1", "numvfs": 8,' \
+               '"use_dhcp": false, "promisc": false, "link_mode":' \
+               '"switchdev"}'
+        pf = objects.object_from_json(json.loads(data))
+        self.assertEqual("p6p1", pf.name)
+        self.assertEqual(8, pf.numvfs)
+        self.assertEqual("off", pf.promisc)
+        self.assertFalse(pf.use_dhcp)
+        self.assertEqual("switchdev", pf.link_mode)
+
+    def test_from_json_link_mode_invalid(self):
+        data = '{"type": "sriov_pf", "name": "p6p1", "numvfs": 8,' \
+               '"use_dhcp": false, "promisc": false, "link_mode":' \
+               '"none_switchdev"}'
+        err = self.assertRaises(objects.InvalidConfigException,
+                                objects.object_from_json,
+                                json.loads(data))
+        expected = 'Expecting link_mode to match legacy/switchdev'
+        self.assertIn(expected, six.text_type(err))
+
 
 class TestSriovVF(base.TestCase):
 

os_net_config/tests/test_utils.py

@@ -123,8 +123,8 @@ class TestUtils(base.TestCase):
         contents = utils.get_file_data(sriov_config._SRIOV_CONFIG_FILE)
         sriov_pf_map = yaml.load(contents) if contents else []
         self.assertEqual(1, len(sriov_pf_map))
-        test_sriov_pf_map = [{'device_type': 'pf', 'name': 'eth1',
+        test_sriov_pf_map = [{'device_type': 'pf', 'link_mode': 'legacy',
-                              'numvfs': 10}]
+                              'name': 'eth1', 'numvfs': 10}]
         self.assertListEqual(test_sriov_pf_map, sriov_pf_map)
 
     def test_update_sriov_pf_map_new_with_promisc(self):
@@ -132,16 +132,18 @@ class TestUtils(base.TestCase):
         contents = utils.get_file_data(sriov_config._SRIOV_CONFIG_FILE)
         sriov_pf_map = yaml.load(contents) if contents else []
         self.assertEqual(1, len(sriov_pf_map))
-        test_sriov_pf_map = [{'device_type': 'pf', 'name': 'eth1',
+        test_sriov_pf_map = [{'device_type': 'pf', 'link_mode': 'legacy',
-                              'numvfs': 10, 'promisc': 'off'}]
+                              'name': 'eth1', 'numvfs': 10, 'promisc': 'off'}]
         self.assertListEqual(test_sriov_pf_map, sriov_pf_map)
 
     def test_update_sriov_pf_map_exist(self):
-        pf_initial = [{'device_type': 'pf', 'name': 'eth1', 'numvfs': 10}]
+        pf_initial = [{'device_type': 'pf', 'link_mode': 'legacy',
+                       'name': 'eth1', 'numvfs': 10}]
         utils.write_yaml_config(sriov_config._SRIOV_CONFIG_FILE, pf_initial)
 
         utils.update_sriov_pf_map('eth1', 20, False)
-        pf_final = [{'device_type': 'pf', 'name': 'eth1', 'numvfs': 20}]
+        pf_final = [{'device_type': 'pf', 'link_mode': 'legacy',
+                     'name': 'eth1', 'numvfs': 20}]
         contents = utils.get_file_data(sriov_config._SRIOV_CONFIG_FILE)
 
         pf_map = yaml.load(contents) if contents else []
@@ -149,13 +151,13 @@ class TestUtils(base.TestCase):
         self.assertListEqual(pf_final, pf_map)
 
     def test_update_sriov_pf_map_exist_with_promisc(self):
-        pf_initial = [{'device_type': 'pf', 'name': 'eth1', 'numvfs': 10,
+        pf_initial = [{'device_type': 'pf', 'link_mode': 'legacy',
-                       'promisc': 'on'}]
+                       'name': 'eth1', 'numvfs': 10, 'promisc': 'on'}]
         utils.write_yaml_config(sriov_config._SRIOV_CONFIG_FILE, pf_initial)
 
         utils.update_sriov_pf_map('eth1', 20, False)
-        pf_final = [{'device_type': 'pf', 'name': 'eth1', 'numvfs': 20,
+        pf_final = [{'device_type': 'pf', 'link_mode': 'legacy',
-                     'promisc': 'on'}]
+                     'name': 'eth1', 'numvfs': 20, 'promisc': 'on'}]
         contents = utils.get_file_data(sriov_config._SRIOV_CONFIG_FILE)
 
         pf_map = yaml.load(contents) if contents else []

os_net_config/utils.py

@@ -414,7 +414,8 @@ def _get_dpdk_mac_address(name):
             return item['mac_address']
 
 
-def update_sriov_pf_map(ifname, numvfs, noop, promisc=None):
+def update_sriov_pf_map(ifname, numvfs, noop, promisc=None,
+                        link_mode='legacy'):
     if not noop:
         sriov_map = _get_sriov_map()
         for item in sriov_map:
@@ -422,6 +423,7 @@ def update_sriov_pf_map(ifname, numvfs, noop, promisc=None):
                 item['numvfs'] = numvfs
                 if promisc is not None:
                     item['promisc'] = promisc
+                item['link_mode'] = link_mode
                 break
         else:
             new_item = {}
@@ -430,6 +432,7 @@ def update_sriov_pf_map(ifname, numvfs, noop, promisc=None):
             new_item['numvfs'] = numvfs
             if promisc is not None:
                 new_item['promisc'] = promisc
+            new_item['link_mode'] = link_mode
             sriov_map.append(new_item)
 
         write_yaml_config(sriov_config._SRIOV_CONFIG_FILE, sriov_map)