openstack
/
os-net-config
Code Issues Proposed changes

Fix failure in dpdk driver binding with VF during reboot 

On a start/reboot, os-net sriov_config.service is run to
create VFs for nic-partitioned devices. If vfio-pci driver
is bound to any of the VFs, the VF initialisation doesn't occur properly.
The VF creation has to be completed before driverctl vfio-pci
binding AND all network interface configs. Since the order of
sriov_config service or driverctl could not be set due to cyclic
dependencies, the driverctl --nosave is used for every reboots
for VFs that needs driver override. This is required in case of
DPDK - NIC Partitioning.

Conflicts:
       os_net_config/common.py
       os_net_config/utils.py
       os_net_config/sriov_config.py
       os_net_config/tests/test_utils.py
Changes are done to exclude refactoring backport done specifically
for vDPA. Also backporting only the required changes with required
factoring.

Change-Id: I3b3712eedf6d909f5d65ecbb1763f9dc11b04c31
(cherry picked from commit 9ef27075eb)
This commit is contained in:
Karthik S 2022-03-03 13:52:27 +00:00
parent dc5ca6da7d
commit 61294bcb03
7 changed files with 163 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
os_net_config/common.py Normal file
View File

@ -0,0 +1,95 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# Common functions and variables meant to be shared across various modules
# As opposed to utils, this is meant to be imported from anywhere. We can't
# import anything from os_net_config here.
import logging
import logging.handlers
import os
from oslo_concurrency import processutils
_SYS_BUS_PCI_DEV = '/sys/bus/pci/devices'
MLNX_VENDOR_ID = "0x15b3"
logger = logging.getLogger(__name__)
class OvsDpdkBindException(ValueError):
pass
def get_pci_dev_path(pci_address, path=None):
if not path:
path = ""
elif path.startswith("_"):
path = path[1:]
return os.path.join(_SYS_BUS_PCI_DEV, pci_address, path)
def get_interface_driver_by_pci_address(pci_address):
try:
uevent = get_pci_dev_path(pci_address, 'uevent')
with open(uevent, 'r') as f:
out = f.read().strip()
for line in out.split('\n'):
if 'DRIVER' in line:
driver = line.split('=')
if len(driver) == 2:
return driver[1]
except IOError:
return
def is_vf(pci_address):
# If DPDK drivers are bound on a VF, then the path common.SYS_CLASS_NET
# wouldn't exist. Instead we look for the path
# /sys/bus/pci/devices/<PCI addr>/physfn to understand if the device
# is actually a VF. This path could be used by VFs not bound with
# DPDK drivers as well
vf_path_check = _SYS_BUS_PCI_DEV + '/%s/physfn' % pci_address
is_sriov_vf = os.path.isdir(vf_path_check)
return is_sriov_vf
def set_driverctl_override(pci_address, driver):
if driver is None:
logger.info(f"Driver override is not required for device"
"{pci_address}")
return False
iface_driver = get_interface_driver_by_pci_address(pci_address)
if iface_driver == driver:
logger.info(f"Driver {driver} is already bound to the device"
"{pci_address}")
return False
try:
if is_vf(pci_address):
out, err = processutils.execute('driverctl', '--nosave',
'set-override', pci_address,
driver)
else:
out, err = processutils.execute('driverctl', 'set-override',
pci_address, driver)
if err:
msg = f"Failed to bind dpdk interface {pci_address} err - {err}"
raise OvsDpdkBindException(msg)
except processutils.ProcessExecutionError:
msg = f"Failed to bind interface {pci_address} with dpdk"
raise OvsDpdkBindException(msg)
return err

19
os_net_config/objects.py
View File

@ -1033,7 +1033,8 @@ class LinuxBond(_BaseOpts):
macaddr=iface.macaddr, promisc=iface.promisc,
pci_address=iface.pci_address,
min_tx_rate=iface.min_tx_rate,
max_tx_rate=iface.max_tx_rate)
max_tx_rate=iface.max_tx_rate,
driver=None)
@staticmethod
def from_json(json):
@ -1119,7 +1120,8 @@ class OvsBond(_BaseOpts):
trust=iface.trust, state=iface.state,
macaddr=iface.macaddr, promisc=iface.promisc,
min_tx_rate=iface.min_tx_rate,
max_tx_rate=iface.max_tx_rate)
max_tx_rate=iface.max_tx_rate,
driver=None)
@staticmethod
def from_json(json):
@ -1328,7 +1330,7 @@ class OvsDpdkPort(_BaseOpts):
self.rx_queue = rx_queue
@staticmethod
def update_vf_config(iface):
def update_vf_config(iface, driver=None):
if iface.trust is None:
logger.info("Trust is not set for VF %s:%d, defaulting to on"
% (iface.device, iface.vfid))
@ -1340,6 +1342,7 @@ class OvsDpdkPort(_BaseOpts):
if iface.promisc is not None:
logger.warning("Promisc can't be changed for ovs_dpdk_port")
iface.promisc = None
logger.info("Overriding the default driver for DPDK")
utils.update_sriov_vf_map(iface.device, iface.vfid, iface.name,
vlan_id=iface.vlan_id, qos=iface.qos,
spoofcheck=iface.spoofcheck,
@ -1347,7 +1350,8 @@ class OvsDpdkPort(_BaseOpts):
macaddr=iface.macaddr, promisc=iface.promisc,
pci_address=iface.pci_address,
min_tx_rate=iface.min_tx_rate,
max_tx_rate=iface.max_tx_rate)
max_tx_rate=iface.max_tx_rate,
driver=driver)
@staticmethod
def from_json(json):
@ -1379,7 +1383,7 @@ class OvsDpdkPort(_BaseOpts):
# be set in the interface part of DPDK Port
members.append(iface)
elif isinstance(iface, SriovVF):
OvsDpdkPort.update_vf_config(iface)
OvsDpdkPort.update_vf_config(iface, driver)
members.append(iface)
else:
msg = 'Unsupported OVS DPDK Port member type'
@ -1453,6 +1457,8 @@ class SriovVF(_BaseOpts):
self.macaddr = macaddr
self.promisc = promisc
self.pci_address = pci_address
self.driver = None
utils.update_sriov_vf_map(device, self.vfid, name,
vlan_id=self.vlan_id,
qos=self.qos,
@ -1463,7 +1469,8 @@ class SriovVF(_BaseOpts):
promisc=promisc,
pci_address=pci_address,
min_tx_rate=min_tx_rate,
max_tx_rate=max_tx_rate)
max_tx_rate=max_tx_rate,
driver=self.driver)
@staticmethod
def get_on_off(config):

7
os_net_config/sriov_config.py
View File

@ -31,6 +31,7 @@ import sys
import time
import yaml
from os_net_config import common
from os_net_config import sriov_bind_config
from oslo_concurrency import processutils
@ -42,7 +43,6 @@ _IFUP_LOCAL_FILE = '/sbin/ifup-local'
_RESET_SRIOV_RULES_FILE = '/etc/udev/rules.d/70-tripleo-reset-sriov.rules'
_ALLOCATE_VFS_FILE = '/etc/sysconfig/allocate_vfs'
_MLNX_DRIVER = "mlx5_core"
MLNX_VENDOR_ID = "0x15b3"
MAX_RETRIES = 10
PF_FUNC_RE = re.compile(r"\.(\d+)$", 0)
@ -306,7 +306,7 @@ def configure_sriov_pf(execution_from_cli=False, restart_openvswitch=False):
vendor_id = get_vendor_id(item['name'])
if (item.get('link_mode') == "switchdev" and
vendor_id == MLNX_VENDOR_ID):
vendor_id == common.MLNX_VENDOR_ID):
logger.info(f"{item['name']}: Mellanox card")
vf_pcis_list = get_vf_pcis_list(item['name'])
mlnx_vfs_pcis_list += vf_pcis_list
@ -650,6 +650,9 @@ def configure_sriov_vf():
if 'promisc' in item:
run_ip_config_cmd('ip', 'link', 'set', 'dev', item['name'],
'promisc', item['promisc'])
if 'driver' in item:
common.set_driverctl_override(item['pci_address'],
item['driver'])
def parse_opts(argv):

15
os_net_config/tests/test_impl_ifcfg.py
View File

@ -1498,8 +1498,8 @@ DOMAIN="openstack.local subdomain.openstack.local"
def test_update_sriov_vf_map(pf_name, vfid, vf_name, vlan_id=None,
qos=None, spoofcheck=None, trust=None,
state=None, macaddr=None, promisc=None,
pci_address=None,
min_tx_rate=0, max_tx_rate=0):
pci_address=None, min_tx_rate=0,
max_tx_rate=0, driver=None):
self.assertEqual(pf_name, 'eth2')
self.assertEqual(vfid, 7)
self.assertEqual(vlan_id, 0)
@ -1511,6 +1511,7 @@ DOMAIN="openstack.local subdomain.openstack.local"
self.assertEqual(state, None)
self.assertEqual(macaddr, None)
self.assertEqual(pci_address, '0000:79:10.2')
self.assertEqual(driver, None)
self.stub_out('os_net_config.utils.update_sriov_vf_map',
test_update_sriov_vf_map)
@ -1548,8 +1549,8 @@ NETMASK=255.255.255.0
def test_update_sriov_vf_map(pf_name, vfid, vf_name, vlan_id=None,
qos=None, spoofcheck=None, trust=None,
state=None, macaddr=None, promisc=None,
pci_address=None,
min_tx_rate=0, max_tx_rate=0):
pci_address=None, min_tx_rate=0,
max_tx_rate=0, driver=None):
self.assertEqual(pf_name, 'eth2')
self.assertEqual(vf_name, 'eth2_7')
self.assertEqual(vfid, 7)
@ -1563,6 +1564,7 @@ NETMASK=255.255.255.0
self.assertEqual(macaddr, "AA:BB:CC:DD:EE:FF")
self.assertTrue(promisc)
self.assertEqual(pci_address, '0000:80:10.1')
self.assertEqual(driver, None)
self.stub_out('os_net_config.utils.update_sriov_vf_map',
test_update_sriov_vf_map)
@ -1604,8 +1606,8 @@ NETMASK=255.255.255.0
def test_update_sriov_vf_map(pf_name, vfid, vf_name, vlan_id=None,
qos=None, spoofcheck=None, trust=None,
state=None, macaddr=None, promisc=None,
pci_address=None,
min_tx_rate=0, max_tx_rate=0):
pci_address=None, min_tx_rate=0,
max_tx_rate=0, driver=None):
self.assertEqual(pf_name, 'eth2')
self.assertEqual(vf_name, 'eth2_7')
self.assertEqual(vfid, 7)
@ -1619,6 +1621,7 @@ NETMASK=255.255.255.0
self.assertEqual(macaddr, "AA:BB:CC:DD:EE:FF")
self.assertFalse(promisc)
self.assertEqual(pci_address, '0000:82:00.2')
self.assertEqual(driver, None)
self.stub_out('os_net_config.utils.update_sriov_vf_map',
test_update_sriov_vf_map)

6
os_net_config/tests/test_objects.py
View File

@ -689,7 +689,8 @@ class TestBridge(base.TestCase):
'vlan_id': 111, 'qos': 1,
'min_tx_rate': 0, 'max_tx_rate': 0,
'spoofcheck': 'off', 'trust': 'on',
'pci_address': '0000:79:10.2'
'pci_address': '0000:79:10.2',
'driver': 'vfio-pci'
}]
def test_get_vf_devname(device, vfid):
@ -745,7 +746,8 @@ class TestBridge(base.TestCase):
'vlan_id': 111, 'qos': 1,
'min_tx_rate': 100, 'max_tx_rate': 500,
'spoofcheck': 'off', 'trust': 'off',
'pci_address': '0000:79:10.2'
'pci_address': '0000:79:10.2',
'driver': 'vfio-pci'
}]
def test_get_vf_devname(device, vfid):

19
os_net_config/tests/test_utils.py
View File

@ -22,6 +22,7 @@ import tempfile
from unittest import mock
import yaml
from os_net_config import common
from os_net_config import objects
from os_net_config import sriov_config
from os_net_config.tests import base
@ -451,7 +452,7 @@ class TestUtils(base.TestCase):
test_get_dpdk_mac_address)
try:
utils.bind_dpdk_interfaces('nic2', 'vfio-pci', False)
except utils.OvsDpdkBindException:
except common.OvsDpdkBindException:
self.fail("Received OvsDpdkBindException unexpectedly")
def test_bind_dpdk_interfaces_fail(self):
@ -468,7 +469,7 @@ class TestUtils(base.TestCase):
self.stub_out('os_net_config.utils._get_dpdk_mac_address',
test_get_dpdk_mac_address)
self.assertRaises(utils.OvsDpdkBindException,
self.assertRaises(common.OvsDpdkBindException,
utils.bind_dpdk_interfaces, 'eth1', 'vfio-pci',
False)
@ -493,7 +494,7 @@ class TestUtils(base.TestCase):
test_get_dpdk_mac_address)
try:
utils.bind_dpdk_interfaces('eth1', 'vfio-pci', False)
except utils.OvsDpdkBindException:
except common.OvsDpdkBindException:
self.fail("Received OvsDpdkBindException unexpectedly")
def test_bind_dpdk_interfaces_fail_invalid_device(self):
@ -518,7 +519,7 @@ class TestUtils(base.TestCase):
self.stub_out('os_net_config.utils._get_dpdk_mac_address',
test_get_dpdk_mac_address)
self.assertRaises(utils.OvsDpdkBindException,
self.assertRaises(common.OvsDpdkBindException,
utils.bind_dpdk_interfaces, 'eth2', 'vfio-pci',
False)
@ -529,7 +530,7 @@ class TestUtils(base.TestCase):
self.stub_out('os_net_config.utils.logger.info', mocked_logger)
try:
utils.bind_dpdk_interfaces('eth1', 'vfio-pci', False)
except utils.OvsDpdkBindException:
except common.OvsDpdkBindException:
self.fail("Received OvsDpdkBindException unexpectedly")
msg = "Driver (vfio-pci) is already bound to the device (eth1)"
mocked_logger.assert_called_with(msg)
@ -602,8 +603,8 @@ class TestUtils(base.TestCase):
tmpdir = tempfile.mkdtemp()
self.stub_out('os_net_config.utils._SYS_CLASS_NET', tmpdir)
tmp_pci_dir = tempfile.mkdtemp()
self.stub_out('os_net_config.utils._SYS_BUS_PCI_DEV', tmp_pci_dir)
physfn_path = utils._SYS_BUS_PCI_DEV + '/0000:05:01.1/physfn'
self.stub_out('os_net_config.common._SYS_BUS_PCI_DEV', tmp_pci_dir)
physfn_path = common._SYS_BUS_PCI_DEV + '/0000:05:01.1/physfn'
os.makedirs(physfn_path)
def test_is_available_nic(interface_name, check_active):
@ -643,8 +644,8 @@ class TestUtils(base.TestCase):
tmpdir = tempfile.mkdtemp()
self.stub_out('os_net_config.utils._SYS_CLASS_NET', tmpdir)
tmp_pci_dir = tempfile.mkdtemp()
self.stub_out('os_net_config.utils._SYS_BUS_PCI_DEV', tmp_pci_dir)
physfn_path = utils._SYS_BUS_PCI_DEV + '/0000:05:01.1/physfn'
self.stub_out('os_net_config.common._SYS_BUS_PCI_DEV', tmp_pci_dir)
physfn_path = common._SYS_BUS_PCI_DEV + '/0000:05:01.1/physfn'
os.makedirs(physfn_path)
def test_is_available_nic(interface_name, check_active):

77
os_net_config/utils.py
View File

@ -22,12 +22,12 @@ import six
import time
import yaml
from os_net_config import common
from os_net_config import sriov_config
from oslo_concurrency import processutils
logger = logging.getLogger(__name__)
_SYS_CLASS_NET = '/sys/class/net'
_SYS_BUS_PCI_DEV = '/sys/bus/pci/devices'
# File to contain the DPDK mapped nics, as nic name will not be available after
# binding driver, which is required for correct nic numbering.
# Format of the file (list mapped nic's details):
@ -44,14 +44,14 @@ _SRIOV_CONFIG_SERVICE_FILE = "/etc/systemd/system/sriov_config.service"
_SRIOV_CONFIG_DEVICE_CONTENT = """[Unit]
Description=SR-IOV numvfs configuration
After=systemd-udev-settle.service openibd.service
Before=openvswitch.service
Before=network-pre.target openvswitch.service
[Service]
Type=oneshot
ExecStart=/usr/bin/os-net-config-sriov
[Install]
WantedBy=multi-user.target
WantedBy=basic.target
"""
# VPP startup operational configuration file. The content of this file will
@ -63,10 +63,6 @@ class InvalidInterfaceException(ValueError):
pass
class OvsDpdkBindException(ValueError):
pass
class VppException(ValueError):
pass
@ -154,19 +150,6 @@ def is_real_nic(interface_name):
return False
def _is_vf(pci_address):
# If DPDK drivers are bound on a VF, then the path _SYS_CLASS_NET
# wouldn't exist. Instead we look for the path
# /sys/bus/pci/devices/<PCI addr>/physfn to understand if the device
# is actually a VF. This path could be used by VFs not bound with
# DPDK drivers as well
vf_path_check = _SYS_BUS_PCI_DEV + '/%s/physfn' % pci_address
is_sriov_vf = os.path.isdir(vf_path_check)
return is_sriov_vf
def _is_vf_by_name(interface_name, check_mapping_file=False):
vf_path_check = _SYS_CLASS_NET + '/%s/device/physfn' % interface_name
is_sriov_vf = os.path.isdir(vf_path_check)
@ -253,7 +236,7 @@ def _ordered_nics(check_active):
# If the DPDK drivers are bound to a VF, the same needs
# to be skipped for the NIC ordering
nic = item['name']
if _is_vf(item['pci_address']):
if common.is_vf(item['pci_address']):
logger.info("%s is a VF, skipping it for NIC ordering" % nic)
elif _is_vf_by_name(nic, True):
logger.info("%s is a VF, skipping it for NIC ordering" % nic)
@ -302,33 +285,23 @@ def bind_dpdk_interfaces(ifname, driver, noop):
processutils.execute('modprobe', 'vfio-pci')
except processutils.ProcessExecutionError:
msg = "Failed to modprobe vfio-pci module"
raise OvsDpdkBindException(msg)
raise common.OvsDpdkBindException(msg)
mac_address = interface_mac(ifname)
vendor_id = get_vendor_id(ifname)
try:
out, err = processutils.execute('driverctl', 'set-override',
pci_address, driver)
if err:
msg = "Failed to bind dpdk interface err - %s" % err
raise OvsDpdkBindException(msg)
else:
_update_dpdk_map(ifname, pci_address, mac_address, driver)
# Not like other nics, beacause mellanox nics keep the
# interface after binding it to dpdk, so we are adding
# ethtool command with 10 attempts after binding the driver
# just to make sure that the interface is initialized
# successfully in order not to fail in each of this cases:
# - get_dpdk_devargs() in case of OvsDpdkPort and
# OvsDpdkBond.
# - bind_dpdk_interface() in case of OvsDpdkBond.
if vendor_id == "0x15b3":
processutils.execute('ethtool', '-i', ifname,
attempts=10)
except processutils.ProcessExecutionError:
msg = "Failed to bind interface %s with dpdk" % ifname
raise OvsDpdkBindException(msg)
err = common.set_driverctl_override(pci_address, driver)
if not err:
_update_dpdk_map(ifname, pci_address, mac_address, driver)
# Not like other nics, beacause mellanox nics keep the
# interface after binding it to dpdk, so we are adding
# ethtool command with 10 attempts after binding the driver
# just to make sure that the interface is initialized
# successfully in order not to fail in each of this cases:
# - get_dpdk_devargs() in case of OvsDpdkPort and
# OvsDpdkBond.
# - bind_dpdk_interface() in case of OvsDpdkBond.
if vendor_id == common.MLNX_VENDOR_ID:
processutils.execute('ethtool', '-i', ifname, attempts=10)
else:
# Check if the pci address is already fetched and stored.
# If the pci address could not be fetched from dpdk_mapping.yaml
@ -336,7 +309,7 @@ def bind_dpdk_interfaces(ifname, driver, noop):
# available nor bound with dpdk.
if not get_stored_pci_address(ifname, noop):
msg = "Interface %s cannot be found" % ifname
raise OvsDpdkBindException(msg)
raise common.OvsDpdkBindException(msg)
else:
logger.info('Interface %(name)s bound to DPDK driver %(driver)s '
'using driverctl command' %
@ -534,7 +507,7 @@ def _get_sriov_map():
def _set_vf_fields(vf_name, vlan_id, qos, spoofcheck, trust, state, macaddr,
promisc, pci_address, min_tx_rate, max_tx_rate):
promisc, pci_address, min_tx_rate, max_tx_rate, driver):
vf_configs = {}
vf_configs['name'] = vf_name
if vlan_id != 0:
@ -553,6 +526,8 @@ def _set_vf_fields(vf_name, vlan_id, qos, spoofcheck, trust, state, macaddr,
vf_configs['macaddr'] = macaddr
vf_configs['promisc'] = promisc
vf_configs['pci_address'] = pci_address
if driver:
vf_configs['driver'] = driver
return vf_configs
@ -565,7 +540,7 @@ def _clear_empty_values(vf_config):
def update_sriov_vf_map(pf_name, vfid, vf_name, vlan_id=0, qos=0,
spoofcheck=None, trust=None, state=None, macaddr=None,
promisc=None, pci_address=None,
min_tx_rate=0, max_tx_rate=0):
min_tx_rate=0, max_tx_rate=0, driver=None):
sriov_map = _get_sriov_map()
for item in sriov_map:
if (item['device_type'] == 'vf' and
@ -573,7 +548,8 @@ def update_sriov_vf_map(pf_name, vfid, vf_name, vlan_id=0, qos=0,
item['device'].get('vfid') == vfid):
item.update(_set_vf_fields(vf_name, vlan_id, qos, spoofcheck,
trust, state, macaddr, promisc,
pci_address, min_tx_rate, max_tx_rate))
pci_address, min_tx_rate, max_tx_rate,
driver))
_clear_empty_values(item)
break
else:
@ -582,7 +558,8 @@ def update_sriov_vf_map(pf_name, vfid, vf_name, vlan_id=0, qos=0,
new_item['device'] = {"name": pf_name, "vfid": vfid}
new_item.update(_set_vf_fields(vf_name, vlan_id, qos, spoofcheck,
trust, state, macaddr, promisc,
pci_address, min_tx_rate, max_tx_rate))
pci_address, min_tx_rate, max_tx_rate,
driver))
_clear_empty_values(new_item)
sriov_map.append(new_item)