From d9e94d859bd67d7b944924ff657dee3deaec6576 Mon Sep 17 00:00:00 2001 From: vcandappa Date: Wed, 13 Oct 2021 19:53:32 +0530 Subject: [PATCH] Add ifcfg-* scripts on boot for Mellanox NIC interface When OvsDpdkPort is configured on compute node using Mellanox NIC interface, the NetworkManager-wait-online service fails after compute boot. This fix creates a seperate ifcfg-* interface with nm_controll and dhcp disabled for these Mellanox ports (used as OvsDpdkPort/DpdkBond) Change-Id: I03d709f59e9550409f42747c4593cd85dd1380c9 (cherry picked from commit e5c50e8cf3c5d9f23ddbcd2123f227d95be79bae) --- os_net_config/impl_ifcfg.py | 22 +++++ os_net_config/tests/test_impl_ifcfg.py | 129 +++++++++++++++++++++++++ os_net_config/utils.py | 8 ++ 3 files changed, 159 insertions(+) diff --git a/os_net_config/impl_ifcfg.py b/os_net_config/impl_ifcfg.py index beaa6fe3..fc7d40b0 100644 --- a/os_net_config/impl_ifcfg.py +++ b/os_net_config/impl_ifcfg.py @@ -958,6 +958,16 @@ class IfcfgNetConfig(os_net_config.NetConfig): logger.debug('ovs dpdk port data: %s' % data) self.interface_data[ovs_dpdk_port.name] = data + """Add an extra ifcfg entry for mellanox NICs, + for the interface which is a member of ovs_dpdk_port + with NM and DHCP disabled + """ + if utils.is_mellanox_interface(ifname): + self.nm_controlled = False + self.use_dhcp = False + dpdk_if_name = objects.Interface(ifname) + self.add_interface(dpdk_if_name) + def add_ovs_dpdk_bond(self, ovs_dpdk_bond): """Add an OvsDPDKBond object to the net config object. @@ -982,6 +992,18 @@ class IfcfgNetConfig(os_net_config.NetConfig): if ovs_dpdk_bond.rules: self._add_rules(ovs_dpdk_bond.name, ovs_dpdk_bond.rules) + """Add an extra ifcfg entry for mellanox NICs, + for all interfaces which is a member of ovs_dpdk_bond + with NM and DHCP disabled + """ + for dpdk_port in ovs_dpdk_bond.members: + ifname = dpdk_port.members[0].name + if utils.is_mellanox_interface(ifname): + self.nm_controlled = False + self.use_dhcp = False + dpdk_if_name = objects.Interface(ifname) + self.add_interface(dpdk_if_name) + def add_sriov_pf(self, sriov_pf): """Add a SriovPF object to the net config object diff --git a/os_net_config/tests/test_impl_ifcfg.py b/os_net_config/tests/test_impl_ifcfg.py index 84b8e34c..70ef05d0 100644 --- a/os_net_config/tests/test_impl_ifcfg.py +++ b/os_net_config/tests/test_impl_ifcfg.py @@ -691,6 +691,8 @@ class TestIfcfgNetConfig(base.TestCase): return "0000:00:09.0" if 'em3' in ifname: return "0000:00:03.0" + if 'em2' in ifname: + return "0000:00:02.0" if 'em1' in ifname: return "0000:00:01.0" @@ -1747,6 +1749,67 @@ OVS_EXTRA="set Interface $DEVICE options:dpdk-devargs=0000:00:09.0" self.provider.bridge_data['br-link']) self.assertEqual(dpdk0_config, self.get_interface_config('dpdk0')) + def test_network_ovs_mellanox_dpdk_bridge_and_port(self): + nic_mapping = {'nic1': 'em1', 'nic2': 'em2', 'nic3': 'em3'} + self.stubbed_mapped_nics = nic_mapping + + def stub_vendor_id_mellanox(ifname): + return True + self.stub_out('os_net_config.utils.is_mellanox_interface', + stub_vendor_id_mellanox) + + interface = objects.Interface(name='nic1') + dpdk_port = objects.OvsDpdkPort(name='dpdk-link1-port', + members=[interface], + driver='mlx5_core') + bridge = objects.OvsUserBridge('br-link1', members=[dpdk_port]) + + def test_bind_dpdk_interfaces(ifname, driver, noop): + self.assertEqual(ifname, 'em1') + self.assertEqual(driver, 'mlx5_core') + self.stub_out('os_net_config.utils.bind_dpdk_interfaces', + test_bind_dpdk_interfaces) + self.stub_out('os_net_config.utils.get_stored_pci_address', + self.stub_get_stored_pci_address) + + self.provider.add_ovs_dpdk_port(dpdk_port) + self.provider.add_ovs_user_bridge(bridge) + br_link_config = """# This file is autogenerated by os-net-config +DEVICE=br-link1 +ONBOOT=yes +HOTPLUG=no +NM_CONTROLLED=no +PEERDNS=no +DEVICETYPE=ovs +TYPE=OVSUserBridge +""" + dpdk0_config = """# This file is autogenerated by os-net-config +DEVICE=dpdk-link1-port +ONBOOT=yes +HOTPLUG=no +NM_CONTROLLED=no +PEERDNS=no +DEVICETYPE=ovs +TYPE=OVSDPDKPort +OVS_BRIDGE=br-link1 +OVS_EXTRA="set Interface $DEVICE options:dpdk-devargs=0000:00:01.0" +""" + self.assertEqual(br_link_config, + self.provider.bridge_data['br-link1']) + self.assertEqual(dpdk0_config, + self.get_interface_config('dpdk-link1-port')) + + if_config = """# This file is autogenerated by os-net-config +DEVICE=em1 +ONBOOT=yes +HOTPLUG=no +NM_CONTROLLED=no +PEERDNS=no +BOOTPROTO=none +""" + self.assertEqual(if_config, + self.get_interface_config('em1')) + def test_network_ovs_dpdk_bridge_and_port_with_mtu_rxqueue(self): nic_mapping = {'nic1': 'eth0', 'nic2': 'eth1', 'nic3': 'eth2'} self.stubbed_mapped_nics = nic_mapping @@ -1832,6 +1895,72 @@ OVS_EXTRA="set Interface dpdk0 options:dpdk-devargs=0000:00:08.0 \ self.assertEqual(dpdk_bond_config, self.get_interface_config('dpdkbond0')) + def test_network_ovs_mellanox_dpdk_bond(self): + nic_mapping = {'nic1': 'em1', 'nic2': 'em2', 'nic3': 'em3'} + self.stubbed_mapped_nics = nic_mapping + + def stub_vendor_id_mellanox(ifname): + return True + self.stub_out('os_net_config.utils.is_mellanox_interface', + stub_vendor_id_mellanox) + + iface0 = objects.Interface(name='nic1') + dpdk0 = objects.OvsDpdkPort(name='dpdk0', + members=[iface0], driver='mlx5_core') + iface1 = objects.Interface(name='nic2') + dpdk1 = objects.OvsDpdkPort(name='dpdk1', + members=[iface1], driver='mlx5_core') + bond = objects.OvsDpdkBond('dpdkbond0', members=[dpdk0, dpdk1]) + bridge = objects.OvsUserBridge('br-link', members=[bond]) + + def test_bind_dpdk_interfaces(ifname, driver, noop): + self.assertIn(ifname, ['em1', 'em2']) + self.assertEqual(driver, 'mlx5_core') + self.stub_out('os_net_config.utils.bind_dpdk_interfaces', + test_bind_dpdk_interfaces) + self.stub_out('os_net_config.utils.get_stored_pci_address', + self.stub_get_stored_pci_address) + + self.provider.add_ovs_dpdk_bond(bond) + self.provider.add_ovs_user_bridge(bridge) + + dpdk_bond_config = """# This file is autogenerated by os-net-config +DEVICE=dpdkbond0 +ONBOOT=yes +HOTPLUG=no +NM_CONTROLLED=no +PEERDNS=no +DEVICETYPE=ovs +TYPE=OVSDPDKBond +OVS_BRIDGE=br-link +BOND_IFACES="dpdk0 dpdk1" +OVS_EXTRA="set Interface dpdk0 options:dpdk-devargs=0000:00:01.0 \ +-- set Interface dpdk1 options:dpdk-devargs=0000:00:02.0" +""" + self.assertEqual(dpdk_bond_config, + self.get_interface_config('dpdkbond0')) + + if_cfg1 = """# This file is autogenerated by os-net-config +DEVICE=em1 +ONBOOT=yes +HOTPLUG=no +NM_CONTROLLED=no +PEERDNS=no +BOOTPROTO=none +""" + if_cfg2 = """# This file is autogenerated by os-net-config +DEVICE=em2 +ONBOOT=yes +HOTPLUG=no +NM_CONTROLLED=no +PEERDNS=no +BOOTPROTO=none +""" + self.assertEqual(if_cfg1, + self.get_interface_config('em1')) + self.assertEqual(if_cfg2, + self.get_interface_config('em2')) + def test_network_ovs_dpdk_bond_with_mtu(self): nic_mapping = {'nic1': 'eth0', 'nic2': 'eth1', 'nic3': 'eth2'} self.stubbed_mapped_nics = nic_mapping diff --git a/os_net_config/utils.py b/os_net_config/utils.py index 7dcd41d7..f868c2c1 100644 --- a/os_net_config/utils.py +++ b/os_net_config/utils.py @@ -403,6 +403,14 @@ def get_device_id(ifname): return +def is_mellanox_interface(ifname): + MLNX_VENDOR_ID = "0x15b3" + vendor_id = get_vendor_id(ifname) + if vendor_id == MLNX_VENDOR_ID: + return True + return False + + def get_interface_driver(ifname): try: uevent = '%s/%s/device/uevent' % (_SYS_CLASS_NET, ifname)