openstack/os-net-config
Code Issues Proposed changes

Fix failure in dpdk driver binding with VF during reboot

Browse Source 
On a start/reboot, os-net sriov_config.service is run to
create VFs for nic-partitioned devices. If vfio-pci driver
is bound to any of the VFs, the VF initialisation doesn't occur properly.
The VF creation has to be completed before driverctl vfio-pci
binding AND all network interface configs. Since the order of
sriov_config service or driverctl could not be set due to cyclic
dependencies, the driverctl --nosave is used for every reboots
for VFs that needs driver override. This is required in case of
DPDK - NIC Partitioning.

Conflict:
    os_net_config/utils.py

Change-Id: I3b3712eedf6d909f5d65ecbb1763f9dc11b04c31
(cherry picked from commit 9ef27075eb)
This commit is contained in:
Karthik S
2022-03-03 13:52:27 +00:00
parent 6626afc240
commit 19307e073e
7 changed files with 160 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
os_net_config/common.py
View File

@@ -57,6 +57,7 @@ DPDK_MAPPING_FILE = '/var/lib/os-net-config/dpdk_mapping.yaml'
SRIOV_CONFIG_FILE = '/var/lib/os-net-config/sriov_config.yaml' SRIOV_CONFIG_FILE = '/var/lib/os-net-config/sriov_config.yaml'
_SYS_BUS_PCI_DEV = '/sys/bus/pci/devices'
SYS_CLASS_NET = '/sys/class/net' SYS_CLASS_NET = '/sys/class/net'
_LOG_FILE = '/var/log/os-net-config.log' _LOG_FILE = '/var/log/os-net-config.log'
MLNX_VENDOR_ID = "0x15b3" MLNX_VENDOR_ID = "0x15b3"
@@ -64,6 +65,10 @@ MLNX_VENDOR_ID = "0x15b3"
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class OvsDpdkBindException(ValueError):
pass
def configure_logger(log_file=False, verbose=False, debug=False): def configure_logger(log_file=False, verbose=False, debug=False):
LOG_FORMAT = ('%(asctime)s.%(msecs)03d %(levelname)s ' LOG_FORMAT = ('%(asctime)s.%(msecs)03d %(levelname)s '
'%(name)s.%(funcName)s %(message)s') '%(name)s.%(funcName)s %(message)s')
@@ -104,6 +109,14 @@ def get_dev_path(ifname, path=None):
return os.path.join(SYS_CLASS_NET, ifname, path) return os.path.join(SYS_CLASS_NET, ifname, path)
def get_pci_dev_path(pci_address, path=None):
if not path:
path = ""
elif path.startswith("_"):
path = path[1:]
return os.path.join(_SYS_BUS_PCI_DEV, pci_address, path)
def get_vendor_id(ifname): def get_vendor_id(ifname):
try: try:
with open(get_dev_path(ifname, "vendor"), 'r') as f: with open(get_dev_path(ifname, "vendor"), 'r') as f:
@@ -141,6 +154,12 @@ def get_sriov_map(pf_name=None):
return sriov_map return sriov_map
def get_dpdk_map():
contents = get_file_data(DPDK_MAPPING_FILE)
dpdk_map = yaml.safe_load(contents) if contents else []
return dpdk_map
def _get_dpdk_mac_address(name): def _get_dpdk_mac_address(name):
contents = get_file_data(DPDK_MAPPING_FILE) contents = get_file_data(DPDK_MAPPING_FILE)
dpdk_map = yaml.safe_load(contents) if contents else [] dpdk_map = yaml.safe_load(contents) if contents else []
@@ -171,11 +190,77 @@ def interface_mac(name):
raise raise
def get_interface_driver_by_pci_address(pci_address):
try:
uevent = get_pci_dev_path(pci_address, 'uevent')
with open(uevent, 'r') as f:
out = f.read().strip()
for line in out.split('\n'):
if 'DRIVER' in line:
driver = line.split('=')
if len(driver) == 2:
return driver[1]
except IOError:
return
def is_mellanox_interface(ifname): def is_mellanox_interface(ifname):
vendor_id = get_vendor_id(ifname) vendor_id = get_vendor_id(ifname)
return vendor_id == MLNX_VENDOR_ID return vendor_id == MLNX_VENDOR_ID
def is_vf(pci_address):
# If DPDK drivers are bound on a VF, then the path common.SYS_CLASS_NET
# wouldn't exist. Instead we look for the path
# /sys/bus/pci/devices/<PCI addr>/physfn to understand if the device
# is actually a VF. This path could be used by VFs not bound with
# DPDK drivers as well
vf_path_check = _SYS_BUS_PCI_DEV + '/%s/physfn' % pci_address
is_sriov_vf = os.path.isdir(vf_path_check)
return is_sriov_vf
def is_vf_by_name(interface_name, check_mapping_file=False):
vf_path_check = get_dev_path(interface_name, 'physfn')
is_sriov_vf = os.path.isdir(vf_path_check)
if not is_sriov_vf and check_mapping_file:
sriov_map = get_sriov_map()
for item in sriov_map:
if (item['name'] == interface_name and
item['device_type'] == 'vf'):
is_sriov_vf = True
return is_sriov_vf
def set_driverctl_override(pci_address, driver):
if driver is None:
logger.info(f"Driver override is not required for device"
"{pci_address}")
return False
iface_driver = get_interface_driver_by_pci_address(pci_address)
if iface_driver == driver:
logger.info(f"Driver {driver} is already bound to the device"
"{pci_address}")
return False
try:
if is_vf(pci_address):
out, err = processutils.execute('driverctl', '--nosave',
'set-override', pci_address,
driver)
else:
out, err = processutils.execute('driverctl', 'set-override',
pci_address, driver)
if err:
msg = f"Failed to bind dpdk interface {pci_address} err - {err}"
raise OvsDpdkBindException(msg)
except processutils.ProcessExecutionError:
msg = f"Failed to bind interface {pci_address} with dpdk"
raise OvsDpdkBindException(msg)
return err
def list_kmods(mods: list) -> list: def list_kmods(mods: list) -> list:
"""Listing Kernel Modules """Listing Kernel Modules

19
os_net_config/objects.py
View File

@@ -1034,7 +1034,8 @@ class LinuxBond(_BaseOpts):
macaddr=iface.macaddr, promisc=iface.promisc, macaddr=iface.macaddr, promisc=iface.promisc,
pci_address=iface.pci_address, pci_address=iface.pci_address,
min_tx_rate=iface.min_tx_rate, min_tx_rate=iface.min_tx_rate,
max_tx_rate=iface.max_tx_rate) max_tx_rate=iface.max_tx_rate,
driver=None)
@staticmethod @staticmethod
def from_json(json): def from_json(json):
@@ -1120,7 +1121,8 @@ class OvsBond(_BaseOpts):
trust=iface.trust, state=iface.state, trust=iface.trust, state=iface.state,
macaddr=iface.macaddr, promisc=iface.promisc, macaddr=iface.macaddr, promisc=iface.promisc,
min_tx_rate=iface.min_tx_rate, min_tx_rate=iface.min_tx_rate,
max_tx_rate=iface.max_tx_rate) max_tx_rate=iface.max_tx_rate,
driver=None)
@staticmethod @staticmethod
def from_json(json): def from_json(json):
@@ -1325,7 +1327,7 @@ class OvsDpdkPort(_BaseOpts):
self.rx_queue = rx_queue self.rx_queue = rx_queue
@staticmethod @staticmethod
def update_vf_config(iface): def update_vf_config(iface, driver=None):
if iface.trust is None: if iface.trust is None:
logger.info("Trust is not set for VF %s:%d, defaulting to on" logger.info("Trust is not set for VF %s:%d, defaulting to on"
% (iface.device, iface.vfid)) % (iface.device, iface.vfid))
@@ -1337,6 +1339,7 @@ class OvsDpdkPort(_BaseOpts):
if iface.promisc is not None: if iface.promisc is not None:
logger.warning("Promisc can't be changed for ovs_dpdk_port") logger.warning("Promisc can't be changed for ovs_dpdk_port")
iface.promisc = None iface.promisc = None
logger.info("Overriding the default driver for DPDK")
utils.update_sriov_vf_map(iface.device, iface.vfid, iface.name, utils.update_sriov_vf_map(iface.device, iface.vfid, iface.name,
vlan_id=iface.vlan_id, qos=iface.qos, vlan_id=iface.vlan_id, qos=iface.qos,
spoofcheck=iface.spoofcheck, spoofcheck=iface.spoofcheck,
@@ -1344,7 +1347,8 @@ class OvsDpdkPort(_BaseOpts):
macaddr=iface.macaddr, promisc=iface.promisc, macaddr=iface.macaddr, promisc=iface.promisc,
pci_address=iface.pci_address, pci_address=iface.pci_address,
min_tx_rate=iface.min_tx_rate, min_tx_rate=iface.min_tx_rate,
max_tx_rate=iface.max_tx_rate) max_tx_rate=iface.max_tx_rate,
driver=driver)
@staticmethod @staticmethod
def from_json(json): def from_json(json):
@@ -1376,7 +1380,7 @@ class OvsDpdkPort(_BaseOpts):
# be set in the interface part of DPDK Port # be set in the interface part of DPDK Port
members.append(iface) members.append(iface)
elif isinstance(iface, SriovVF): elif isinstance(iface, SriovVF):
OvsDpdkPort.update_vf_config(iface) OvsDpdkPort.update_vf_config(iface, driver)
members.append(iface) members.append(iface)
else: else:
msg = 'Unsupported OVS DPDK Port member type' msg = 'Unsupported OVS DPDK Port member type'
@@ -1450,6 +1454,8 @@ class SriovVF(_BaseOpts):
self.macaddr = macaddr self.macaddr = macaddr
self.promisc = promisc self.promisc = promisc
self.pci_address = pci_address self.pci_address = pci_address
self.driver = None
utils.update_sriov_vf_map(device, self.vfid, name, utils.update_sriov_vf_map(device, self.vfid, name,
vlan_id=self.vlan_id, vlan_id=self.vlan_id,
qos=self.qos, qos=self.qos,
@@ -1460,7 +1466,8 @@ class SriovVF(_BaseOpts):
promisc=promisc, promisc=promisc,
pci_address=pci_address, pci_address=pci_address,
min_tx_rate=min_tx_rate, min_tx_rate=min_tx_rate,
max_tx_rate=max_tx_rate) max_tx_rate=max_tx_rate,
driver=self.driver)
@staticmethod @staticmethod
def get_on_off(config): def get_on_off(config):

3
os_net_config/sriov_config.py
View File

@@ -718,6 +718,9 @@ def configure_sriov_vf():
if 'promisc' in item: if 'promisc' in item:
run_ip_config_cmd('ip', 'link', 'set', 'dev', item['name'], run_ip_config_cmd('ip', 'link', 'set', 'dev', item['name'],
'promisc', item['promisc']) 'promisc', item['promisc'])
if 'driver' in item:
common.set_driverctl_override(item['pci_address'],
item['driver'])
def parse_opts(argv): def parse_opts(argv):

15
os_net_config/tests/test_impl_ifcfg.py
View File

@@ -1498,8 +1498,8 @@ DOMAIN="openstack.local subdomain.openstack.local"
def test_update_sriov_vf_map(pf_name, vfid, vf_name, vlan_id=None, def test_update_sriov_vf_map(pf_name, vfid, vf_name, vlan_id=None,
qos=None, spoofcheck=None, trust=None, qos=None, spoofcheck=None, trust=None,
state=None, macaddr=None, promisc=None, state=None, macaddr=None, promisc=None,
pci_address=None, pci_address=None, min_tx_rate=0,
min_tx_rate=0, max_tx_rate=0): max_tx_rate=0, driver=None):
self.assertEqual(pf_name, 'eth2') self.assertEqual(pf_name, 'eth2')
self.assertEqual(vfid, 7) self.assertEqual(vfid, 7)
self.assertEqual(vlan_id, 0) self.assertEqual(vlan_id, 0)
@@ -1511,6 +1511,7 @@ DOMAIN="openstack.local subdomain.openstack.local"
self.assertEqual(state, None) self.assertEqual(state, None)
self.assertEqual(macaddr, None) self.assertEqual(macaddr, None)
self.assertEqual(pci_address, '0000:79:10.2') self.assertEqual(pci_address, '0000:79:10.2')
self.assertEqual(driver, None)
self.stub_out('os_net_config.utils.update_sriov_vf_map', self.stub_out('os_net_config.utils.update_sriov_vf_map',
test_update_sriov_vf_map) test_update_sriov_vf_map)
@@ -1548,8 +1549,8 @@ NETMASK=255.255.255.0
def test_update_sriov_vf_map(pf_name, vfid, vf_name, vlan_id=None, def test_update_sriov_vf_map(pf_name, vfid, vf_name, vlan_id=None,
qos=None, spoofcheck=None, trust=None, qos=None, spoofcheck=None, trust=None,
state=None, macaddr=None, promisc=None, state=None, macaddr=None, promisc=None,
pci_address=None, pci_address=None, min_tx_rate=0,
min_tx_rate=0, max_tx_rate=0): max_tx_rate=0, driver=None):
self.assertEqual(pf_name, 'eth2') self.assertEqual(pf_name, 'eth2')
self.assertEqual(vf_name, 'eth2_7') self.assertEqual(vf_name, 'eth2_7')
self.assertEqual(vfid, 7) self.assertEqual(vfid, 7)
@@ -1563,6 +1564,7 @@ NETMASK=255.255.255.0
self.assertEqual(macaddr, "AA:BB:CC:DD:EE:FF") self.assertEqual(macaddr, "AA:BB:CC:DD:EE:FF")
self.assertTrue(promisc) self.assertTrue(promisc)
self.assertEqual(pci_address, '0000:80:10.1') self.assertEqual(pci_address, '0000:80:10.1')
self.assertEqual(driver, None)
self.stub_out('os_net_config.utils.update_sriov_vf_map', self.stub_out('os_net_config.utils.update_sriov_vf_map',
test_update_sriov_vf_map) test_update_sriov_vf_map)
@@ -1604,8 +1606,8 @@ NETMASK=255.255.255.0
def test_update_sriov_vf_map(pf_name, vfid, vf_name, vlan_id=None, def test_update_sriov_vf_map(pf_name, vfid, vf_name, vlan_id=None,
qos=None, spoofcheck=None, trust=None, qos=None, spoofcheck=None, trust=None,
state=None, macaddr=None, promisc=None, state=None, macaddr=None, promisc=None,
pci_address=None, pci_address=None, min_tx_rate=0,
min_tx_rate=0, max_tx_rate=0): max_tx_rate=0, driver=None):
self.assertEqual(pf_name, 'eth2') self.assertEqual(pf_name, 'eth2')
self.assertEqual(vf_name, 'eth2_7') self.assertEqual(vf_name, 'eth2_7')
self.assertEqual(vfid, 7) self.assertEqual(vfid, 7)
@@ -1619,6 +1621,7 @@ NETMASK=255.255.255.0
self.assertEqual(macaddr, "AA:BB:CC:DD:EE:FF") self.assertEqual(macaddr, "AA:BB:CC:DD:EE:FF")
self.assertFalse(promisc) self.assertFalse(promisc)
self.assertEqual(pci_address, '0000:82:00.2') self.assertEqual(pci_address, '0000:82:00.2')
self.assertEqual(driver, None)
self.stub_out('os_net_config.utils.update_sriov_vf_map', self.stub_out('os_net_config.utils.update_sriov_vf_map',
test_update_sriov_vf_map) test_update_sriov_vf_map)

6
os_net_config/tests/test_objects.py
View File

@@ -687,7 +687,8 @@ class TestBridge(base.TestCase):
'vlan_id': 111, 'qos': 1, 'vlan_id': 111, 'qos': 1,
'min_tx_rate': 0, 'max_tx_rate': 0, 'min_tx_rate': 0, 'max_tx_rate': 0,
'spoofcheck': 'off', 'trust': 'on', 'spoofcheck': 'off', 'trust': 'on',
'pci_address': '0000:79:10.2' 'pci_address': '0000:79:10.2',
'driver': 'vfio-pci'
}] }]
def test_get_vf_devname(device, vfid): def test_get_vf_devname(device, vfid):
@@ -743,7 +744,8 @@ class TestBridge(base.TestCase):
'vlan_id': 111, 'qos': 1, 'vlan_id': 111, 'qos': 1,
'min_tx_rate': 100, 'max_tx_rate': 500, 'min_tx_rate': 100, 'max_tx_rate': 500,
'spoofcheck': 'off', 'trust': 'off', 'spoofcheck': 'off', 'trust': 'off',
'pci_address': '0000:79:10.2' 'pci_address': '0000:79:10.2',
'driver': 'vfio-pci'
}] }]
def test_get_vf_devname(device, vfid): def test_get_vf_devname(device, vfid):

28
os_net_config/tests/test_utils.py
View File

@@ -439,7 +439,7 @@ class TestUtils(base.TestCase):
'mac_address': '01:02:03:04:05:06', 'mac_address': '01:02:03:04:05:06',
'driver': 'vfio-pci'}] 'driver': 'vfio-pci'}]
self.stub_out('os_net_config.utils._get_dpdk_map', test_get_dpdk_map) self.stub_out('os_net_config.common.get_dpdk_map', test_get_dpdk_map)
pci = utils.get_stored_pci_address('eth1', False) pci = utils.get_stored_pci_address('eth1', False)
self.assertEqual('0000:00:09.0', pci) self.assertEqual('0000:00:09.0', pci)
@@ -447,7 +447,7 @@ class TestUtils(base.TestCase):
def test_get_dpdk_map(): def test_get_dpdk_map():
return [] return []
self.stub_out('os_net_config.utils._get_dpdk_map', test_get_dpdk_map) self.stub_out('os_net_config.common.get_dpdk_map', test_get_dpdk_map)
pci = utils.get_stored_pci_address('eth1', False) pci = utils.get_stored_pci_address('eth1', False)
self.assertEqual(None, pci) self.assertEqual(None, pci)
@@ -492,7 +492,7 @@ class TestUtils(base.TestCase):
test_get_dpdk_mac_address) test_get_dpdk_mac_address)
try: try:
utils.bind_dpdk_interfaces('nic2', 'vfio-pci', False) utils.bind_dpdk_interfaces('nic2', 'vfio-pci', False)
except utils.OvsDpdkBindException: except common.OvsDpdkBindException:
self.fail("Received OvsDpdkBindException unexpectedly") self.fail("Received OvsDpdkBindException unexpectedly")
def test_bind_dpdk_interfaces_fail(self): def test_bind_dpdk_interfaces_fail(self):
@@ -509,7 +509,7 @@ class TestUtils(base.TestCase):
self.stub_out('os_net_config.common._get_dpdk_mac_address', self.stub_out('os_net_config.common._get_dpdk_mac_address',
test_get_dpdk_mac_address) test_get_dpdk_mac_address)
self.assertRaises(utils.OvsDpdkBindException, self.assertRaises(common.OvsDpdkBindException,
utils.bind_dpdk_interfaces, 'eth1', 'vfio-pci', utils.bind_dpdk_interfaces, 'eth1', 'vfio-pci',
False) False)
@@ -528,13 +528,13 @@ class TestUtils(base.TestCase):
'mac_address': '01:02:03:04:05:06', 'mac_address': '01:02:03:04:05:06',
'driver': 'vfio-pci'}] 'driver': 'vfio-pci'}]
self.stub_out('os_net_config.utils._get_dpdk_map', test_get_dpdk_map) self.stub_out('os_net_config.common.get_dpdk_map', test_get_dpdk_map)
self.stub_out('oslo_concurrency.processutils.execute', test_execute) self.stub_out('oslo_concurrency.processutils.execute', test_execute)
self.stub_out('os_net_config.utils_get_dpdk_mac_address', self.stub_out('os_net_config.utils_get_dpdk_mac_address',
test_get_dpdk_mac_address) test_get_dpdk_mac_address)
try: try:
utils.bind_dpdk_interfaces('eth1', 'vfio-pci', False) utils.bind_dpdk_interfaces('eth1', 'vfio-pci', False)
except utils.OvsDpdkBindException: except common.OvsDpdkBindException:
self.fail("Received OvsDpdkBindException unexpectedly") self.fail("Received OvsDpdkBindException unexpectedly")
def test_bind_dpdk_interfaces_fail_invalid_device(self): def test_bind_dpdk_interfaces_fail_invalid_device(self):
@@ -552,14 +552,14 @@ class TestUtils(base.TestCase):
'mac_address': '01:02:03:04:05:06', 'mac_address': '01:02:03:04:05:06',
'driver': 'vfio-pci'}] 'driver': 'vfio-pci'}]
self.stub_out('os_net_config.utils_get_dpdk_map', self.stub_out('os_net_config.common.get_dpdk_map',
test_get_dpdk_map) test_get_dpdk_map)
self.stub_out('oslo_concurrency.processutils.execute', self.stub_out('oslo_concurrency.processutils.execute',
test_execute) test_execute)
self.stub_out('os_net_config.utils._get_dpdk_mac_address', self.stub_out('os_net_config.utils._get_dpdk_mac_address',
test_get_dpdk_mac_address) test_get_dpdk_mac_address)
self.assertRaises(utils.OvsDpdkBindException, self.assertRaises(common.OvsDpdkBindException,
utils.bind_dpdk_interfaces, 'eth2', 'vfio-pci', utils.bind_dpdk_interfaces, 'eth2', 'vfio-pci',
False) False)
@@ -570,7 +570,7 @@ class TestUtils(base.TestCase):
self.stub_out('os_net_config.utils.logger.info', mocked_logger) self.stub_out('os_net_config.utils.logger.info', mocked_logger)
try: try:
utils.bind_dpdk_interfaces('eth1', 'vfio-pci', False) utils.bind_dpdk_interfaces('eth1', 'vfio-pci', False)
except utils.OvsDpdkBindException: except common.OvsDpdkBindException:
self.fail("Received OvsDpdkBindException unexpectedly") self.fail("Received OvsDpdkBindException unexpectedly")
msg = "Driver (vfio-pci) is already bound to the device (eth1)" msg = "Driver (vfio-pci) is already bound to the device (eth1)"
mocked_logger.assert_called_with(msg) mocked_logger.assert_called_with(msg)
@@ -643,8 +643,8 @@ class TestUtils(base.TestCase):
tmpdir = tempfile.mkdtemp() tmpdir = tempfile.mkdtemp()
self.stub_out('os_net_config.common.SYS_CLASS_NET', tmpdir) self.stub_out('os_net_config.common.SYS_CLASS_NET', tmpdir)
tmp_pci_dir = tempfile.mkdtemp() tmp_pci_dir = tempfile.mkdtemp()
self.stub_out('os_net_config.utils._SYS_BUS_PCI_DEV', tmp_pci_dir) self.stub_out('os_net_config.common._SYS_BUS_PCI_DEV', tmp_pci_dir)
physfn_path = utils._SYS_BUS_PCI_DEV + '/0000:05:01.1/physfn' physfn_path = common._SYS_BUS_PCI_DEV + '/0000:05:01.1/physfn'
os.makedirs(physfn_path) os.makedirs(physfn_path)
def test_is_available_nic(interface_name, check_active): def test_is_available_nic(interface_name, check_active):
@@ -684,8 +684,8 @@ class TestUtils(base.TestCase):
tmpdir = tempfile.mkdtemp() tmpdir = tempfile.mkdtemp()
self.stub_out('os_net_config.common.SYS_CLASS_NET', tmpdir) self.stub_out('os_net_config.common.SYS_CLASS_NET', tmpdir)
tmp_pci_dir = tempfile.mkdtemp() tmp_pci_dir = tempfile.mkdtemp()
self.stub_out('os_net_config.utils._SYS_BUS_PCI_DEV', tmp_pci_dir) self.stub_out('os_net_config.common._SYS_BUS_PCI_DEV', tmp_pci_dir)
physfn_path = utils._SYS_BUS_PCI_DEV + '/0000:05:01.1/physfn' physfn_path = common._SYS_BUS_PCI_DEV + '/0000:05:01.1/physfn'
os.makedirs(physfn_path) os.makedirs(physfn_path)
def test_is_available_nic(interface_name, check_active): def test_is_available_nic(interface_name, check_active):
@@ -914,7 +914,7 @@ dpdk {
'mac_address': '01:02:03:04:05:06', 'mac_address': '01:02:03:04:05:06',
'driver': 'vfio-pci'}] 'driver': 'vfio-pci'}]
self.stub_out('os_net_config.utils._get_dpdk_map', test_get_dpdk_map) self.stub_out('os_net_config.common.get_dpdk_map', test_get_dpdk_map)
def test_execute(name, *args, **kwargs): def test_execute(name, *args, **kwargs):
return None, None return None, None

106
os_net_config/utils.py
View File

@@ -26,7 +26,6 @@ from os_net_config import sriov_config
from oslo_concurrency import processutils from oslo_concurrency import processutils
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
_SYS_BUS_PCI_DEV = '/sys/bus/pci/devices'
# sriov_config service shall be created and enabled so that the various # sriov_config service shall be created and enabled so that the various
# SR-IOV PF and VF configurations shall be done during reboot as well using # SR-IOV PF and VF configurations shall be done during reboot as well using
# sriov_config.py installed in path /usr/bin/os-net-config-sriov # sriov_config.py installed in path /usr/bin/os-net-config-sriov
@@ -34,14 +33,14 @@ _SRIOV_CONFIG_SERVICE_FILE = "/etc/systemd/system/sriov_config.service"
_SRIOV_CONFIG_DEVICE_CONTENT = """[Unit] _SRIOV_CONFIG_DEVICE_CONTENT = """[Unit]
Description=SR-IOV numvfs configuration Description=SR-IOV numvfs configuration
After=systemd-udev-settle.service openibd.service After=systemd-udev-settle.service openibd.service
Before=openvswitch.service Before=network-pre.target openvswitch.service
[Service] [Service]
Type=oneshot Type=oneshot
ExecStart=/usr/bin/os-net-config-sriov ExecStart=/usr/bin/os-net-config-sriov
[Install] [Install]
WantedBy=multi-user.target WantedBy=basic.target
""" """
# VPP startup operational configuration file. The content of this file will # VPP startup operational configuration file. The content of this file will
@@ -53,10 +52,6 @@ class InvalidInterfaceException(ValueError):
pass pass
class OvsDpdkBindException(ValueError):
pass
class VppException(ValueError): class VppException(ValueError):
pass pass
@@ -110,31 +105,6 @@ def is_real_nic(interface_name):
return False return False
def _is_vf(pci_address):
# If DPDK drivers are bound on a VF, then the path common.SYS_CLASS_NET
# wouldn't exist. Instead we look for the path
# /sys/bus/pci/devices/<PCI addr>/physfn to understand if the device
# is actually a VF. This path could be used by VFs not bound with
# DPDK drivers as well
vf_path_check = _SYS_BUS_PCI_DEV + '/%s/physfn' % pci_address
is_sriov_vf = os.path.isdir(vf_path_check)
return is_sriov_vf
def _is_vf_by_name(interface_name, check_mapping_file=False):
vf_path_check = common.get_dev_path(interface_name, 'physfn')
is_sriov_vf = os.path.isdir(vf_path_check)
if not is_sriov_vf and check_mapping_file:
sriov_map = common.get_sriov_map()
for item in sriov_map:
if (item['name'] == interface_name and
item['device_type'] == 'vf'):
is_sriov_vf = True
return is_sriov_vf
def _is_available_nic(interface_name, check_active=True): def _is_available_nic(interface_name, check_active=True):
try: try:
if interface_name == 'lo': if interface_name == 'lo':
@@ -155,7 +125,7 @@ def _is_available_nic(interface_name, check_active=True):
# the nic numbering. All the VFs will have a reference to the PF with # the nic numbering. All the VFs will have a reference to the PF with
# directory name as 'physfn', if this directory is present it should be # directory name as 'physfn', if this directory is present it should be
# ignored. # ignored.
if _is_vf_by_name(interface_name): if common.is_vf_by_name(interface_name):
return False return False
# nic is available # nic is available
@@ -210,9 +180,9 @@ def _ordered_nics(check_active):
# If the DPDK drivers are bound to a VF, the same needs # If the DPDK drivers are bound to a VF, the same needs
# to be skipped for the NIC ordering # to be skipped for the NIC ordering
nic = item['name'] nic = item['name']
if _is_vf(item['pci_address']): if common.is_vf(item['pci_address']):
logger.info("%s is a VF, skipping it for NIC ordering" % nic) logger.info("%s is a VF, skipping it for NIC ordering" % nic)
elif _is_vf_by_name(nic, True): elif common.is_vf_by_name(nic, True):
logger.info("%s is a VF, skipping it for NIC ordering" % nic) logger.info("%s is a VF, skipping it for NIC ordering" % nic)
elif _is_embedded_nic(nic): elif _is_embedded_nic(nic):
logger.info("%s is an embedded DPDK bound nic" % nic) logger.info("%s is an embedded DPDK bound nic" % nic)
@@ -247,7 +217,7 @@ def bind_dpdk_interfaces(ifname, driver, noop):
if common.is_mellanox_interface(ifname) and 'vfio-pci' in driver: if common.is_mellanox_interface(ifname) and 'vfio-pci' in driver:
msg = ("For Mellanox NIC %s, the default driver vfio-pci " msg = ("For Mellanox NIC %s, the default driver vfio-pci "
"needs to be overridden" % ifname) "needs to be overridden" % ifname)
raise OvsDpdkBindException(msg) raise common.OvsDpdkBindException(msg)
iface_driver = get_interface_driver(ifname) iface_driver = get_interface_driver(ifname)
if iface_driver == driver: if iface_driver == driver:
@@ -264,33 +234,23 @@ def bind_dpdk_interfaces(ifname, driver, noop):
processutils.execute('modprobe', 'vfio-pci') processutils.execute('modprobe', 'vfio-pci')
except processutils.ProcessExecutionError: except processutils.ProcessExecutionError:
msg = "Failed to modprobe vfio-pci module" msg = "Failed to modprobe vfio-pci module"
raise OvsDpdkBindException(msg) raise common.OvsDpdkBindException(msg)
mac_address = common.interface_mac(ifname) mac_address = common.interface_mac(ifname)
vendor_id = common.get_vendor_id(ifname) vendor_id = common.get_vendor_id(ifname)
try: err = common.set_driverctl_override(pci_address, driver)
out, err = processutils.execute('driverctl', 'set-override', if not err:
pci_address, driver) _update_dpdk_map(ifname, pci_address, mac_address, driver)
if err: # Not like other nics, beacause mellanox nics keep the
msg = "Failed to bind dpdk interface err - %s" % err # interface after binding it to dpdk, so we are adding
raise OvsDpdkBindException(msg) # ethtool command with 10 attempts after binding the driver
else: # just to make sure that the interface is initialized
_update_dpdk_map(ifname, pci_address, mac_address, driver) # successfully in order not to fail in each of this cases:
# Not like other nics, beacause mellanox nics keep the # - get_dpdk_devargs() in case of OvsDpdkPort and
# interface after binding it to dpdk, so we are adding # OvsDpdkBond.
# ethtool command with 10 attempts after binding the driver # - bind_dpdk_interface() in case of OvsDpdkBond.
# just to make sure that the interface is initialized if vendor_id == common.MLNX_VENDOR_ID:
# successfully in order not to fail in each of this cases: processutils.execute('ethtool', '-i', ifname, attempts=10)
# - get_dpdk_devargs() in case of OvsDpdkPort and
# OvsDpdkBond.
# - bind_dpdk_interface() in case of OvsDpdkBond.
if vendor_id == common.MLNX_VENDOR_ID:
processutils.execute('ethtool', '-i', ifname,
attempts=10)
except processutils.ProcessExecutionError:
msg = "Failed to bind interface %s with dpdk" % ifname
raise OvsDpdkBindException(msg)
else: else:
# Check if the pci address is already fetched and stored. # Check if the pci address is already fetched and stored.
# If the pci address could not be fetched from dpdk_mapping.yaml # If the pci address could not be fetched from dpdk_mapping.yaml
@@ -298,7 +258,7 @@ def bind_dpdk_interfaces(ifname, driver, noop):
# available nor bound with dpdk. # available nor bound with dpdk.
if not get_stored_pci_address(ifname, noop): if not get_stored_pci_address(ifname, noop):
msg = "Interface %s cannot be found" % ifname msg = "Interface %s cannot be found" % ifname
raise OvsDpdkBindException(msg) raise common.OvsDpdkBindException(msg)
else: else:
logger.info('Interface %(name)s bound to DPDK driver %(driver)s ' logger.info('Interface %(name)s bound to DPDK driver %(driver)s '
'using driverctl command' % 'using driverctl command' %
@@ -327,7 +287,7 @@ def get_pci_address(ifname, noop):
def get_stored_pci_address(ifname, noop): def get_stored_pci_address(ifname, noop):
if not noop: if not noop:
dpdk_map = _get_dpdk_map() dpdk_map = common.get_dpdk_map()
for dpdk_nic in dpdk_map: for dpdk_nic in dpdk_map:
if dpdk_nic['name'] == ifname: if dpdk_nic['name'] == ifname:
return dpdk_nic['pci_address'] return dpdk_nic['pci_address']
@@ -376,7 +336,7 @@ def get_dpdk_devargs(ifname, noop):
# in dpdk_mapping.yaml file, so we need to get their pci # in dpdk_mapping.yaml file, so we need to get their pci
# address with ethtool. # address with ethtool.
dpdk_devargs = get_pci_address(ifname, noop) dpdk_devargs = get_pci_address(ifname, noop)
elif _is_vf_by_name(ifname): elif common.is_vf_by_name(ifname):
# For Mellanox devices the VFs bound with DPDK shall # For Mellanox devices the VFs bound with DPDK shall
# be treated the same as VFs of other devices # be treated the same as VFs of other devices
dpdk_devargs = get_pci_address(ifname, noop) dpdk_devargs = get_pci_address(ifname, noop)
@@ -396,7 +356,7 @@ def get_dpdk_devargs(ifname, noop):
# is stored persistently in DPDK_MAPPING_FILE and is used to for nic numbering # is stored persistently in DPDK_MAPPING_FILE and is used to for nic numbering
# on subsequent runs of os-net-config. # on subsequent runs of os-net-config.
def _update_dpdk_map(ifname, pci_address, mac_address, driver): def _update_dpdk_map(ifname, pci_address, mac_address, driver):
dpdk_map = _get_dpdk_map() dpdk_map = common.get_dpdk_map()
for item in dpdk_map: for item in dpdk_map:
if item['pci_address'] == pci_address: if item['pci_address'] == pci_address:
item['name'] = ifname item['name'] = ifname
@@ -414,12 +374,6 @@ def _update_dpdk_map(ifname, pci_address, mac_address, driver):
write_yaml_config(common.DPDK_MAPPING_FILE, dpdk_map) write_yaml_config(common.DPDK_MAPPING_FILE, dpdk_map)
def _get_dpdk_map():
contents = common.get_file_data(common.DPDK_MAPPING_FILE)
dpdk_map = yaml.safe_load(contents) if contents else []
return dpdk_map
def update_sriov_pf_map(ifname, numvfs, noop, promisc=None, def update_sriov_pf_map(ifname, numvfs, noop, promisc=None,
link_mode='legacy', vdpa=False, steering_mode=None): link_mode='legacy', vdpa=False, steering_mode=None):
if not noop: if not noop:
@@ -455,7 +409,7 @@ def update_sriov_pf_map(ifname, numvfs, noop, promisc=None,
def _set_vf_fields(vf_name, vlan_id, qos, spoofcheck, trust, state, macaddr, def _set_vf_fields(vf_name, vlan_id, qos, spoofcheck, trust, state, macaddr,
promisc, pci_address, min_tx_rate, max_tx_rate): promisc, pci_address, min_tx_rate, max_tx_rate, driver):
vf_configs = {} vf_configs = {}
vf_configs['name'] = vf_name vf_configs['name'] = vf_name
if vlan_id != 0: if vlan_id != 0:
@@ -474,6 +428,8 @@ def _set_vf_fields(vf_name, vlan_id, qos, spoofcheck, trust, state, macaddr,
vf_configs['macaddr'] = macaddr vf_configs['macaddr'] = macaddr
vf_configs['promisc'] = promisc vf_configs['promisc'] = promisc
vf_configs['pci_address'] = pci_address vf_configs['pci_address'] = pci_address
if driver:
vf_configs['driver'] = driver
return vf_configs return vf_configs
@@ -486,7 +442,7 @@ def _clear_empty_values(vf_config):
def update_sriov_vf_map(pf_name, vfid, vf_name, vlan_id=0, qos=0, def update_sriov_vf_map(pf_name, vfid, vf_name, vlan_id=0, qos=0,
spoofcheck=None, trust=None, state=None, macaddr=None, spoofcheck=None, trust=None, state=None, macaddr=None,
promisc=None, pci_address=None, promisc=None, pci_address=None,
min_tx_rate=0, max_tx_rate=0): min_tx_rate=0, max_tx_rate=0, driver=None):
sriov_map = common.get_sriov_map() sriov_map = common.get_sriov_map()
for item in sriov_map: for item in sriov_map:
if (item['device_type'] == 'vf' and if (item['device_type'] == 'vf' and
@@ -494,7 +450,8 @@ def update_sriov_vf_map(pf_name, vfid, vf_name, vlan_id=0, qos=0,
item['device'].get('vfid') == vfid): item['device'].get('vfid') == vfid):
item.update(_set_vf_fields(vf_name, vlan_id, qos, spoofcheck, item.update(_set_vf_fields(vf_name, vlan_id, qos, spoofcheck,
trust, state, macaddr, promisc, trust, state, macaddr, promisc,
pci_address, min_tx_rate, max_tx_rate)) pci_address, min_tx_rate, max_tx_rate,
driver))
_clear_empty_values(item) _clear_empty_values(item)
break break
else: else:
@@ -503,7 +460,8 @@ def update_sriov_vf_map(pf_name, vfid, vf_name, vlan_id=0, qos=0,
new_item['device'] = {"name": pf_name, "vfid": vfid} new_item['device'] = {"name": pf_name, "vfid": vfid}
new_item.update(_set_vf_fields(vf_name, vlan_id, qos, spoofcheck, new_item.update(_set_vf_fields(vf_name, vlan_id, qos, spoofcheck,
trust, state, macaddr, promisc, trust, state, macaddr, promisc,
pci_address, min_tx_rate, max_tx_rate)) pci_address, min_tx_rate, max_tx_rate,
driver))
_clear_empty_values(new_item) _clear_empty_values(new_item)
sriov_map.append(new_item) sriov_map.append(new_item)