From a0477bccebe84be1b82f1424528433ef358457f0 Mon Sep 17 00:00:00 2001 From: waleed mousa Date: Wed, 30 Jan 2019 02:39:06 -0500 Subject: [PATCH] Configure switchdev mode in SR-IOV in os-net-config While sriov vfs are created now using os-net-config, so doing some modification to configure switchdev mode there Closes-Bug: #1818034 (cherry picked from commit 3dcad07311be36161529069fdd14107aa2142ee2) Conflicts: os_net_config/objects.py os_net_config/schema.yaml Change-Id: I7c85ab380dc4f0377a65fc954fcce3e3d39f5029 Signed-off-by: waleed mousa --- .../samples/sriov_pf_switchdev.json | 84 ++++++++++++++++++ .../samples/sriov_pf_switchdev.yaml | 87 +++++++++++++++++++ os_net_config/impl_ifcfg.py | 3 +- os_net_config/objects.py | 10 ++- os_net_config/schema.yaml | 7 ++ os_net_config/sriov_config.py | 86 ++++++++++++++++++ os_net_config/tests/test_impl_ifcfg.py | 12 ++- os_net_config/tests/test_objects.py | 22 +++++ os_net_config/tests/test_utils.py | 22 ++--- os_net_config/utils.py | 5 +- 10 files changed, 321 insertions(+), 17 deletions(-) create mode 100644 etc/os-net-config/samples/sriov_pf_switchdev.json create mode 100644 etc/os-net-config/samples/sriov_pf_switchdev.yaml diff --git a/etc/os-net-config/samples/sriov_pf_switchdev.json b/etc/os-net-config/samples/sriov_pf_switchdev.json new file mode 100644 index 00000000..8b3cb014 --- /dev/null +++ b/etc/os-net-config/samples/sriov_pf_switchdev.json @@ -0,0 +1,84 @@ +{ + "network_config": [ + { + "type": "sriov_pf", + "name": "p2p1", + "numvfs": 10, + "use_dhcp": false, + "promisc": true, + "link_mode": "switchdev" + }, + { + "type": "ovs_bridge", + "name": "br-pfs", + "members": [ + { + "type": "sriov_pf", + "name": "p2p2", + "numvfs": 10, + "promisc": true, + "use_dhcp": false, + "link_mode": "switchdev" + } + ], + "use_dhcp": true + }, + { + "type": "ovs_bridge", + "name": "br-bond", + "use_dhcp": true, + "members": [ + { + "type": "ovs_bond", + "name": "bond_pf", + "ovs_options": "bond_mode=active-backup", + "members": [ + { + "type": "sriov_pf", + "name": "p5p1", + "numvfs": 10, + "primary": true, + "promisc": true, + "use_dhcp": false, + "link_mode": "switchdev" + }, + { + "type": "sriov_pf", + "name": "p5p2", + "numvfs": 10, + "promisc": true, + "use_dhcp": false, + "link_mode": "switchdev" + } + ] + } + ] + }, + { + "type": "linux_bond", + "name": "bond_lnx", + "use_dhcp": true, + "bonding_options": "mode=active-backup", + "members": [ + { + "type": "sriov_pf", + "name": "p6p1", + "numvfs": 10, + "primary": true, + "promisc": true, + "use_dhcp": false, + "link_mode": "switchdev" + }, + { + "type": "sriov_pf", + "name": "p6p2", + "numvfs": 10, + "promisc": true, + "use_dhcp": false, + "link_mode": "switchdev" + + } + ] + } + ] +} diff --git a/etc/os-net-config/samples/sriov_pf_switchdev.yaml b/etc/os-net-config/samples/sriov_pf_switchdev.yaml new file mode 100644 index 00000000..6962ed41 --- /dev/null +++ b/etc/os-net-config/samples/sriov_pf_switchdev.yaml @@ -0,0 +1,87 @@ +network_config: + # sriov_pf type shall be used to configure the PF's of NICs. + # The numvfs configured for the PF's shall be set on the sriov_numvfs of the + # sysfs for the corresponding NIC and the persistence of the same across reboots + # shall be handled. + # The link_mode configured the mode of the sriov_pf which can be "switchdev" + # or legacy. + # In case fo switchdev, you will enable the hardware offloading and VF-LAG + # capabilities + # For now, if you configured link_mode with "switchdev" you will not be able + # to use any of its sriov_vfs, the vfs will be only used for vms. + - + type: sriov_pf + # nic name or nic number of the NIC that needs to be configured for SRIOV + name: p2p1 + # number of VFs required on the particular NIC + numvfs: 10 + addresses: + - ip_netmask: 192.0.2.1/24 + # Allow all the traffic received. Default: true + promisc: true + # The mode of sriov_pf which: + # - switchdev + # - legacy (default) + link_mode: switchdev + + - + type: ovs_bridge + name: br-pfs + use_dhcp: true + members: + - + type: sriov_pf + name: p2p2 + numvfs: 10 + promisc: true + use_dhcp: false + link_mode: switchdev + + - + type: ovs_bridge + name: br-bond + use_dhcp: true + members: + - + type: ovs_bond + name: bond_pf + ovs_options: "bond_mode=active-backup" + members: + - + type: sriov_pf + name: p5p1 + numvfs: 10 + primary: true + promisc: true + use_dhcp: false + link_mode: switchdev + - + type: sriov_pf + name: p5p2 + numvfs: 10 + promisc: true + use_dhcp: false + link_mode: switchdev + + - + # VF-LAG capability, which means that the vfs will be bonded in hardware + type: linux_bond + name: bond_lnx + use_dhcp: true + bonding_options: "mode=active-backup" + members: + - + type: sriov_pf + name: p6p1 + numvfs: 10 + primary: true + promisc: true + use_dhcp: false + link_mode: switchdev + - + type: sriov_pf + name: p6p2 + numvfs: 10 + promisc: true + use_dhcp: false + link_mode: switchdev diff --git a/os_net_config/impl_ifcfg.py b/os_net_config/impl_ifcfg.py index e89edac5..23354b9b 100644 --- a/os_net_config/impl_ifcfg.py +++ b/os_net_config/impl_ifcfg.py @@ -857,7 +857,8 @@ class IfcfgNetConfig(os_net_config.NetConfig): data = self._add_common(sriov_pf) logger.debug('sriov pf data: %s' % data) utils.update_sriov_pf_map(sriov_pf.name, sriov_pf.numvfs, - self.noop, promisc=sriov_pf.promisc) + self.noop, promisc=sriov_pf.promisc, + link_mode=sriov_pf.link_mode) self.interface_data[sriov_pf.name] = data def add_sriov_vf(self, sriov_vf): diff --git a/os_net_config/objects.py b/os_net_config/objects.py index 5fc37323..bfdfd2f0 100644 --- a/os_net_config/objects.py +++ b/os_net_config/objects.py @@ -1317,7 +1317,7 @@ class SriovPF(_BaseOpts): addresses=None, routes=None, mtu=None, primary=False, nic_mapping=None, persist_mapping=False, defroute=True, dhclient_args=None, dns_servers=None, nm_controlled=False, - onboot=True, members=None, promisc=None): + onboot=True, members=None, promisc=None, link_mode='legacy'): addresses = addresses or [] routes = routes or [] dns_servers = dns_servers or [] @@ -1333,6 +1333,7 @@ class SriovPF(_BaseOpts): else: self.name = name self.promisc = promisc + self.link_mode = link_mode @staticmethod def get_on_off(config): @@ -1350,8 +1351,13 @@ class SriovPF(_BaseOpts): # SR-IOV PF - promisc: on (default) promisc = json.get('promisc', True) promisc = SriovPF.get_on_off(promisc) + link_mode = json.get('link_mode', 'legacy') + if link_mode not in ['legacy', 'switchdev']: + msg = 'Expecting link_mode to match legacy/switchdev' + raise InvalidConfigException(msg) opts = _BaseOpts.base_opts_from_json(json) - return SriovPF(name, numvfs, *opts, promisc=promisc) + return SriovPF(name, numvfs, *opts, promisc=promisc, + link_mode=link_mode) class OvsDpdkBond(_BaseOpts): diff --git a/os_net_config/schema.yaml b/os_net_config/schema.yaml index 8e0a6bf1..6c778947 100644 --- a/os_net_config/schema.yaml +++ b/os_net_config/schema.yaml @@ -33,6 +33,11 @@ definitions: sriov_vf_state_string: type: string pattern: "^(auto|enable|disable)$" + sriov_link_mode_or_param: + oneOf: + - type: string + pattern: "^(legacy|switchdev)$" + - $ref: "#/definitions/param" # MAC address type mac_address_string: type: string @@ -291,6 +296,8 @@ definitions: $ref: "#/definitions/bool_or_param" onboot: $ref: "#/definitions/bool_or_param" + link_mode: + $ref: "#/definitions/sriov_link_mode_or_param" required: - type - name diff --git a/os_net_config/sriov_config.py b/os_net_config/sriov_config.py index 8e867101..2269e798 100644 --- a/os_net_config/sriov_config.py +++ b/os_net_config/sriov_config.py @@ -25,6 +25,7 @@ import argparse import logging import os import pyudev +import re from six.moves import queue as Queue import sys import yaml @@ -32,6 +33,7 @@ import yaml from oslo_concurrency import processutils logger = logging.getLogger(__name__) +_SYS_CLASS_NET = '/sys/class/net' # Create a queue for passing the udev network events vf_queue = Queue.Queue() @@ -94,6 +96,8 @@ def configure_sriov_pf(): observer.start() sriov_map = _get_sriov_map() + MLNX_UNBIND_FILE_PATH = "/sys/bus/pci/drivers/mlx5_core/unbind" + MLNX_VENDOR_ID = "0x15b3" for item in sriov_map: if item['device_type'] == 'pf': _pf_interface_up(item) @@ -108,6 +112,19 @@ def configure_sriov_pf(): raise SRIOVNumvfsException(msg) # Wait for the creation of VFs for each PF _wait_for_vf_creation(item['name'], item['numvfs']) + # Configure switchdev mode + vendor_id = get_vendor_id(item['name']) + if (item.get('link_mode') == "switchdev" and + vendor_id == MLNX_VENDOR_ID): + vf_pcis_list = get_vf_pcis_list(item['name']) + for vf_pci in vf_pcis_list: + vf_pci_path = "/sys/bus/pci/devices/%s/driver" % vf_pci + if os.path.exists(vf_pci_path): + with open(MLNX_UNBIND_FILE_PATH, 'w') as f: + f.write("%s" % vf_pci) + configure_switchdev(item['name']) + if_up_interface(item['name']) + observer.stop() @@ -139,6 +156,30 @@ def _wait_for_vf_creation(pf_name, numvfs): logger.info("Required VFs are created for PF %s" % pf_name) +def configure_switchdev(pf_name): + pf_pci = get_pf_pci(pf_name) + pf_device_id = get_pf_device_id(pf_name) + if pf_device_id == "0x1013" or pf_device_id == "0x1015": + try: + processutils.execute('/usr/sbin/devlink', 'dev', 'eswitch', 'set', + 'pci/%s' % pf_pci, 'inline-mode', 'transport') + except processutils.ProcessExecutionError: + logger.error("Failed to set inline-mode to transport") + raise + try: + processutils.execute('/usr/sbin/devlink', 'dev', 'eswitch', 'set', + 'pci/%s' % pf_pci, 'mode', 'switchdev') + except processutils.ProcessExecutionError: + logger.error("Failed to set mode to switchdev") + raise + try: + processutils.execute('/usr/sbin/ethtool', '-K', pf_name, + 'hw-tc-offload', 'on') + except processutils.ProcessExecutionError: + logger.error("Failed to enable hw-tc-offload") + raise + + def run_ip_config_cmd(*cmd, **kwargs): logger.info("Running %s" % ' '.join(cmd)) try: @@ -156,6 +197,51 @@ def _pf_interface_up(pf_device): run_ip_config_cmd('ip', 'link', 'set', 'dev', pf_device['name'], 'up') +def get_vendor_id(ifname): + try: + with open(os.path.join(_SYS_CLASS_NET, ifname, "device/vendor"), + 'r') as f: + out = f.read().strip() + return out + except IOError: + return + + +def get_pf_pci(pf_name): + pf_pci_path = os.path.join(_SYS_CLASS_NET, pf_name, "device/uevent") + pf_info = get_file_data(pf_pci_path) + pf_pci = re.search(r'PCI_SLOT_NAME=(.*)', pf_info, re.MULTILINE).group(1) + return pf_pci + + +def get_pf_device_id(pf_name): + pf_device_path = os.path.join(_SYS_CLASS_NET, pf_name, "device/device") + pf_device_id = get_file_data(pf_device_path).strip() + return pf_device_id + + +def get_vf_pcis_list(pf_name): + vf_pcis_list = [] + listOfPfFiles = os.listdir(os.path.join(_SYS_CLASS_NET, pf_name, + "device")) + for pf_file in listOfPfFiles: + if pf_file.startswith("virtfn"): + vf_info = get_file_data(os.path.join(_SYS_CLASS_NET, pf_name, + "device", pf_file, "uevent")) + vf_pcis_list.append(re.search(r'PCI_SLOT_NAME=(.*)', + vf_info, re.MULTILINE).group(1)) + return vf_pcis_list + + +def if_up_interface(device): + logger.info("Running /sbin/ifup %s" % device) + try: + processutils.execute('/sbin/ifup', device) + except processutils.ProcessExecutionError: + logger.error("Failed to ifup %s" % device) + raise + + def configure_sriov_vf(): sriov_map = _get_sriov_map() for item in sriov_map: diff --git a/os_net_config/tests/test_impl_ifcfg.py b/os_net_config/tests/test_impl_ifcfg.py index be5d4680..c6c97801 100644 --- a/os_net_config/tests/test_impl_ifcfg.py +++ b/os_net_config/tests/test_impl_ifcfg.py @@ -1309,10 +1309,12 @@ NETMASK=255.255.255.0 pf = objects.SriovPF(name='nic3', numvfs=10) - def test_update_sriov_pf_map(name, numvfs, noop, promisc=None): + def test_update_sriov_pf_map(name, numvfs, noop, promisc=None, + link_mode='legacy'): self.assertEqual(name, 'eth2') self.assertEqual(numvfs, 10) self.assertEqual(promisc, None) + self.assertEqual(link_mode, 'legacy') self.stub_out('os_net_config.utils.update_sriov_pf_map', test_update_sriov_pf_map) self.provider.add_sriov_pf(pf) @@ -1332,10 +1334,12 @@ BOOTPROTO=none pf = objects.SriovPF(name='nic3', numvfs=10, promisc=True) - def test_update_sriov_pf_map(name, numvfs, noop, promisc=None): + def test_update_sriov_pf_map(name, numvfs, noop, promisc=None, + link_mode='legacy'): self.assertEqual(name, 'eth2') self.assertEqual(numvfs, 10) self.assertTrue(promisc) + self.assertEqual(link_mode, 'legacy') self.stub_out('os_net_config.utils.update_sriov_pf_map', test_update_sriov_pf_map) self.provider.add_sriov_pf(pf) @@ -1355,10 +1359,12 @@ BOOTPROTO=none pf = objects.SriovPF(name='nic3', numvfs=10, promisc=False) - def test_update_sriov_pf_map(name, numvfs, noop, promisc=None): + def test_update_sriov_pf_map(name, numvfs, noop, promisc=None, + link_mode='legacy'): self.assertEqual(name, 'eth2') self.assertEqual(numvfs, 10) self.assertFalse(promisc) + self.assertEqual(link_mode, 'legacy') self.stub_out('os_net_config.utils.update_sriov_pf_map', test_update_sriov_pf_map) self.provider.add_sriov_pf(pf) diff --git a/os_net_config/tests/test_objects.py b/os_net_config/tests/test_objects.py index cd71de34..5e4efcd7 100644 --- a/os_net_config/tests/test_objects.py +++ b/os_net_config/tests/test_objects.py @@ -1675,6 +1675,7 @@ class TestSriovPF(base.TestCase): self.assertEqual(16, pf.numvfs) self.assertEqual("off", pf.promisc) self.assertFalse(pf.use_dhcp) + self.assertEqual("legacy", pf.link_mode) def test_from_json_numvfs_nic1(self): def dummy_mapped_nics(nic_mapping=None): @@ -1702,6 +1703,27 @@ class TestSriovPF(base.TestCase): self.assertFalse(pf.use_dhcp) self.assertEqual('on', pf.promisc) + def test_from_json_link_mode(self): + data = '{"type": "sriov_pf", "name": "p6p1", "numvfs": 8,' \ + '"use_dhcp": false, "promisc": false, "link_mode":' \ + '"switchdev"}' + pf = objects.object_from_json(json.loads(data)) + self.assertEqual("p6p1", pf.name) + self.assertEqual(8, pf.numvfs) + self.assertEqual("off", pf.promisc) + self.assertFalse(pf.use_dhcp) + self.assertEqual("switchdev", pf.link_mode) + + def test_from_json_link_mode_invalid(self): + data = '{"type": "sriov_pf", "name": "p6p1", "numvfs": 8,' \ + '"use_dhcp": false, "promisc": false, "link_mode":' \ + '"none_switchdev"}' + err = self.assertRaises(objects.InvalidConfigException, + objects.object_from_json, + json.loads(data)) + expected = 'Expecting link_mode to match legacy/switchdev' + self.assertIn(expected, six.text_type(err)) + class TestSriovVF(base.TestCase): diff --git a/os_net_config/tests/test_utils.py b/os_net_config/tests/test_utils.py index 5de3012b..275fea62 100644 --- a/os_net_config/tests/test_utils.py +++ b/os_net_config/tests/test_utils.py @@ -123,8 +123,8 @@ class TestUtils(base.TestCase): contents = utils.get_file_data(sriov_config._SRIOV_CONFIG_FILE) sriov_pf_map = yaml.load(contents) if contents else [] self.assertEqual(1, len(sriov_pf_map)) - test_sriov_pf_map = [{'device_type': 'pf', 'name': 'eth1', - 'numvfs': 10}] + test_sriov_pf_map = [{'device_type': 'pf', 'link_mode': 'legacy', + 'name': 'eth1', 'numvfs': 10}] self.assertListEqual(test_sriov_pf_map, sriov_pf_map) def test_update_sriov_pf_map_new_with_promisc(self): @@ -132,16 +132,18 @@ class TestUtils(base.TestCase): contents = utils.get_file_data(sriov_config._SRIOV_CONFIG_FILE) sriov_pf_map = yaml.load(contents) if contents else [] self.assertEqual(1, len(sriov_pf_map)) - test_sriov_pf_map = [{'device_type': 'pf', 'name': 'eth1', - 'numvfs': 10, 'promisc': 'off'}] + test_sriov_pf_map = [{'device_type': 'pf', 'link_mode': 'legacy', + 'name': 'eth1', 'numvfs': 10, 'promisc': 'off'}] self.assertListEqual(test_sriov_pf_map, sriov_pf_map) def test_update_sriov_pf_map_exist(self): - pf_initial = [{'device_type': 'pf', 'name': 'eth1', 'numvfs': 10}] + pf_initial = [{'device_type': 'pf', 'link_mode': 'legacy', + 'name': 'eth1', 'numvfs': 10}] utils.write_yaml_config(sriov_config._SRIOV_CONFIG_FILE, pf_initial) utils.update_sriov_pf_map('eth1', 20, False) - pf_final = [{'device_type': 'pf', 'name': 'eth1', 'numvfs': 20}] + pf_final = [{'device_type': 'pf', 'link_mode': 'legacy', + 'name': 'eth1', 'numvfs': 20}] contents = utils.get_file_data(sriov_config._SRIOV_CONFIG_FILE) pf_map = yaml.load(contents) if contents else [] @@ -149,13 +151,13 @@ class TestUtils(base.TestCase): self.assertListEqual(pf_final, pf_map) def test_update_sriov_pf_map_exist_with_promisc(self): - pf_initial = [{'device_type': 'pf', 'name': 'eth1', 'numvfs': 10, - 'promisc': 'on'}] + pf_initial = [{'device_type': 'pf', 'link_mode': 'legacy', + 'name': 'eth1', 'numvfs': 10, 'promisc': 'on'}] utils.write_yaml_config(sriov_config._SRIOV_CONFIG_FILE, pf_initial) utils.update_sriov_pf_map('eth1', 20, False) - pf_final = [{'device_type': 'pf', 'name': 'eth1', 'numvfs': 20, - 'promisc': 'on'}] + pf_final = [{'device_type': 'pf', 'link_mode': 'legacy', + 'name': 'eth1', 'numvfs': 20, 'promisc': 'on'}] contents = utils.get_file_data(sriov_config._SRIOV_CONFIG_FILE) pf_map = yaml.load(contents) if contents else [] diff --git a/os_net_config/utils.py b/os_net_config/utils.py index d21cd076..8cd0ae9e 100644 --- a/os_net_config/utils.py +++ b/os_net_config/utils.py @@ -414,7 +414,8 @@ def _get_dpdk_mac_address(name): return item['mac_address'] -def update_sriov_pf_map(ifname, numvfs, noop, promisc=None): +def update_sriov_pf_map(ifname, numvfs, noop, promisc=None, + link_mode='legacy'): if not noop: sriov_map = _get_sriov_map() for item in sriov_map: @@ -422,6 +423,7 @@ def update_sriov_pf_map(ifname, numvfs, noop, promisc=None): item['numvfs'] = numvfs if promisc is not None: item['promisc'] = promisc + item['link_mode'] = link_mode break else: new_item = {} @@ -430,6 +432,7 @@ def update_sriov_pf_map(ifname, numvfs, noop, promisc=None): new_item['numvfs'] = numvfs if promisc is not None: new_item['promisc'] = promisc + new_item['link_mode'] = link_mode sriov_map.append(new_item) write_yaml_config(sriov_config._SRIOV_CONFIG_FILE, sriov_map)