Browse Source

Update sriov config service to handle nic partitioned PF

Create VFs using udev rules are required for attaching
and removing PF from a VM. On a reboot, when udev rules
trigger VF creation script and when a VF is attached to
vfio-pci driver, some of the VFs on that PF are not
initialized properly, resulting in VF not found issue.
Though it looks like driver issue, this patch avoids
this scenarios by removing the nic-paritioned VF from
the udev rule, as when one of the VF is attached to host,
the PF cannot be used by the VM.

The function setUp_pf_stubs() was added in the vdpa patch:
    https://review.opendev.org/c/openstack/os-net-config/+/798926/
Since there is a dependency on this API for the tests of this patch,
it has been added here

Change-Id: Ic6738de42b9aa0346f48060c143c5f779b67867e
changes/26/814426/5
vcandappa 9 months ago committed by Vijayalakshmi Candappa
parent
commit
62d4b0f2ac
  1. 26
      os_net_config/sriov_config.py
  2. 141
      os_net_config/tests/test_sriov_config.py
  3. 39
      os_net_config/tests/test_utils.py
  4. 27
      os_net_config/utils.py

26
os_net_config/sriov_config.py

@ -252,6 +252,24 @@ def udev_monitor_stop(observer):
observer.stop()
def is_partitioned_pf(dev_name):
"""Check if any nic-partition(VF) is already used
Given a PF device, returns True if any VFs of this
device are in-use.
"""
sriov_map = _get_sriov_map()
for config in sriov_map:
devtype = config.get('device_type', None)
if devtype == 'vf':
name = config.get('device', {}).get('name')
vf_name = config.get('name')
if dev_name == name:
logger.warning("%s has VF(%s) used by host" % (name, vf_name))
return True
return False
def configure_sriov_pf(execution_from_cli=False, restart_openvswitch=False):
observer = udev_monitor_setup()
udev_monitor_start(observer)
@ -267,8 +285,9 @@ def configure_sriov_pf(execution_from_cli=False, restart_openvswitch=False):
if item.get('link_mode') == "legacy":
# Add a udev rule to configure the VF's when PF's are
# released by a guest
add_udev_rule_for_legacy_sriov_pf(item['name'],
item['numvfs'])
if not is_partitioned_pf(item['name']):
add_udev_rule_for_legacy_sriov_pf(item['name'],
item['numvfs'])
set_numvfs(item['name'], item['numvfs'])
vendor_id = get_vendor_id(item['name'])
if (item.get('link_mode') == "switchdev" and
@ -281,6 +300,7 @@ def configure_sriov_pf(execution_from_cli=False, restart_openvswitch=False):
logger.info("Unbinding %s" % vf_pci)
with open(MLNX_UNBIND_FILE_PATH, 'w') as f:
f.write(vf_pci)
logger.info("%s: Adding udev rules" % item['name'])
# Adding a udev rule to make vf-representors unmanaged by
# NetworkManager
@ -451,6 +471,7 @@ def configure_switchdev(pf_name):
% (pf_pci, exc))
raise
logger.info("Device pci/%s set to switchdev mode." % pf_pci)
# WA to make sure that the uplink_rep is ready after moving to switchdev,
# as moving to switchdev will remove the sriov_pf and create uplink
# representor, so we need to make sure that uplink representor is ready
@ -473,6 +494,7 @@ def configure_smfs_software_steering(pf_name):
'pci/%s' % pf_pci, 'name', 'flow_steering_mode',
'value', 'smfs', 'cmode', 'runtime')
logger.info("Device pci/%s is set to smfs steering mode." % pf_pci)
except processutils.ProcessExecutionError as exc:
logger.warning("Could not set pci/%s to smfs steering mode: %s"
% (pf_pci, exc))

141
os_net_config/tests/test_sriov_config.py

@ -226,6 +226,52 @@ class TestSriovConfig(base.TestCase):
f = open(udev_file.name, 'r')
self.assertEqual(exp_udev_content, f.read())
def setUp_pf_stubs(self, vendor_id="0x8086"):
run_cmd = []
self.setUp_udev_stubs()
def run_ip_config_cmd_stub(*args, **kwargs):
run_cmd.append(' '.join(args))
self.stub_out('os_net_config.sriov_config.run_ip_config_cmd',
run_ip_config_cmd_stub)
def cleanup_puppet_config_stub():
return
self.stub_out('os_net_config.sriov_config.cleanup_puppet_config',
cleanup_puppet_config_stub)
def _wait_for_vf_creation_stub(pf_name, numvfs):
self._save_action('_wait_for_vf_creation_stub')
return
self.stub_out('os_net_config.sriov_config._wait_for_vf_creation',
_wait_for_vf_creation_stub)
def get_vendor_id_stub(ifname):
return vendor_id
self.stub_out('os_net_config.sriov_config.get_vendor_id',
get_vendor_id_stub)
def configure_switchdev_stub(pf_name):
self._save_action('configure_switchdev')
return
self.stub_out('os_net_config.sriov_config.configure_switchdev',
configure_switchdev_stub)
self.set_numvfs = sriov_config.set_numvfs
self.get_numvfs = sriov_config.get_numvfs
def get_numvfs_stub(*args):
self._save_action('get_numvfs')
return self.get_numvfs(*args)
self.stub_out('os_net_config.sriov_config.get_numvfs',
get_numvfs_stub)
def set_numvfs_stub(*args):
self._save_action('set_numvfs')
return self.set_numvfs(*args)
self.stub_out('os_net_config.sriov_config.set_numvfs',
set_numvfs_stub)
def test_configure_sriov_pf(self):
"""Test the numvfs setting for SR-IOV PF
@ -273,6 +319,101 @@ class TestSriovConfig(base.TestCase):
self.assertEqual(10, sriov_config.get_numvfs('p2p1'))
self.assertEqual(12, sriov_config.get_numvfs('p2p2'))
def test_configure_sriov_pf_nicpart(self):
"""Test the udev rules created for legacy mode of SR-IOV PF
In this case, the VF(s) are already bound/attached
"""
self.setUp_pf_stubs()
exp_udev_content = '# This file is autogenerated by os-net-config\n'\
'KERNEL=="p2p2", RUN+="/bin/os-net-config-sriov -n %k:12"\n'
exp_actions = [
'udev_monitor_setup',
'udev_monitor_start',
'set_numvfs',
'get_numvfs',
'_wait_for_vf_creation_stub',
'get_numvfs',
'reload_udev_rules',
'set_numvfs',
'get_numvfs',
'_wait_for_vf_creation_stub',
'get_numvfs',
'udev_monitor_stop',
]
pf_config = [{"device_type": "pf", "name": "p2p1", "numvfs": 10,
"promisc": "on", "link_mode": "legacy"},
{"device_type": "pf", "name": "p2p2", "numvfs": 12,
"promisc": "off", "link_mode": "legacy"},
{"device": {"name": "p2p1", "vfid": 0},
"device_type": "vf", "name": "p2p1v0", "max_tx_rate": 0,
"min_tx_rate": 0, "pci_address": "0000:18:0a.0",
"trust": "on", "spoofcheck": "off"}]
for ifname in ['p2p1', 'p2p2']:
self._write_numvfs(ifname)
self._action_order = []
utils.write_yaml_config(sriov_config._SRIOV_CONFIG_FILE, pf_config)
sriov_config.configure_logger(debug=True)
sriov_config.configure_sriov_pf()
self.assertEqual(exp_actions, self._action_order)
f = open(sriov_config._UDEV_LEGACY_RULE_FILE, 'r')
self.assertEqual(exp_udev_content, f.read())
self.assertEqual(12, sriov_config.get_numvfs('p2p2'))
def test_configure_sriov_pf_non_nicpart(self):
"""Test the udev rules created for legacy mode of SR-IOV PF
In this case, the nic partitioned VF(s) are not attached
"""
self.setUp_pf_stubs()
exp_udev_content = '# This file is autogenerated by os-net-config\n'\
'KERNEL=="p2p1", RUN+="/bin/os-net-config-sriov -n %k:10"\n'\
'KERNEL=="p2p2", RUN+="/bin/os-net-config-sriov -n %k:12"\n'
exp_actions = [
'udev_monitor_setup',
'udev_monitor_start',
'reload_udev_rules',
'set_numvfs',
'get_numvfs',
'_wait_for_vf_creation_stub',
'get_numvfs',
'reload_udev_rules',
'set_numvfs',
'get_numvfs',
'_wait_for_vf_creation_stub',
'get_numvfs',
'udev_monitor_stop',
]
pf_config = [{"device_type": "pf", "name": "p2p1", "numvfs": 10,
"promisc": "on", "link_mode": "legacy"},
{"device_type": "pf", "name": "p2p2", "numvfs": 12,
"promisc": "off", "link_mode": "legacy"},
{"device": {"name": "eno3", "vfid": 1},
"device_type": "vf", "name": "eno3v0", "max_tx_rate": 0,
"min_tx_rate": 0, "pci_address": "0000:18:0e.1",
"trust": "on", "spoofcheck": "off"}]
for ifname in ['p2p1', 'p2p2']:
self._write_numvfs(ifname)
self._action_order = []
utils.write_yaml_config(sriov_config._SRIOV_CONFIG_FILE, pf_config)
sriov_config.configure_logger(debug=True)
sriov_config.configure_sriov_pf()
self.assertEqual(exp_actions, self._action_order)
f = open(sriov_config._UDEV_LEGACY_RULE_FILE, 'r')
self.assertEqual(exp_udev_content, f.read())
self.assertEqual(10, sriov_config.get_numvfs('p2p1'))
self.assertEqual(12, sriov_config.get_numvfs('p2p2'))
def test_cleanup_puppet_config_deprecation(self):
"""Test the cleanup of puppet-tripleo generated config file.

39
os_net_config/tests/test_utils.py

@ -75,6 +75,11 @@ dpdk {
}
'''
udev_content = '# This file is autogenerated by os-net-config\n'\
'KERNEL=="eth1", RUN+="/bin/os-net-config-sriov -n %k:10"\n' \
'KERNEL=="eth2", RUN+="/bin/os-net-config-sriov -n %k:10"\n'
UDEV_FILE = '/tmp/70-os-net-config-sriov.'
class TestUtils(base.TestCase):
@ -83,6 +88,7 @@ class TestUtils(base.TestCase):
rand = str(int(random.random() * 100000))
utils._DPDK_MAPPING_FILE = '/tmp/dpdk_mapping_' + rand + '.yaml'
sriov_config._SRIOV_CONFIG_FILE = '/tmp/sriov_config_' + rand + '.yaml'
sriov_config._UDEV_LEGACY_RULE_FILE = UDEV_FILE + rand
def tearDown(self):
super(TestUtils, self).tearDown()
@ -90,6 +96,8 @@ class TestUtils(base.TestCase):
os.remove(utils._DPDK_MAPPING_FILE)
if os.path.isfile(sriov_config._SRIOV_CONFIG_FILE):
os.remove(sriov_config._SRIOV_CONFIG_FILE)
if os.path.isfile(sriov_config._UDEV_LEGACY_RULE_FILE):
os.remove(sriov_config._UDEV_LEGACY_RULE_FILE)
def test_ordered_active_nics(self):
@ -204,6 +212,37 @@ class TestUtils(base.TestCase):
'device': {"name": "eth1", "vfid": 2}}]
self.assertListEqual(test_sriov_vf_map, sriov_vf_map)
def test_udev_rule_for_sriov_vf(self):
def get_numvfs_stub(pf_name):
return 10
self.stub_out('os_net_config.sriov_config.get_numvfs',
get_numvfs_stub)
utils.update_sriov_pf_map('eth1', 10, False)
utils.update_sriov_pf_map('eth2', 10, False)
utils.update_sriov_vf_map('eth1', 2, 'eth1_2')
contents = utils.get_file_data(sriov_config._SRIOV_CONFIG_FILE)
sriov_vf_map = yaml.safe_load(contents) if contents else []
self.assertEqual(3, len(sriov_vf_map))
test_sriov_vf_map = {'device': {'name': 'eth1', 'vfid': 2},
'device_type': 'vf', 'max_tx_rate': 0,
'min_tx_rate': 0, 'name': 'eth1_2'}
udev_file = open(sriov_config._UDEV_LEGACY_RULE_FILE, "w")
udev_file.write(udev_content)
udev_file.close()
# sriov_config.add_udev_rule_for_legacy_sriov_pf("eth1", 10)
for dev in sriov_vf_map:
if(dev.keys() == test_sriov_vf_map.keys()):
self.assertEqual(test_sriov_vf_map, dev)
expect = '# This file is autogenerated by os-net-config\n'\
'KERNEL=="eth2", RUN+="/bin/os-net-config-sriov -n %k:10"\n'
utils.nicpart_udev_rules_check()
f = open(sriov_config._UDEV_LEGACY_RULE_FILE, 'r')
self.assertEqual(expect, f.read())
f.close()
def test_update_sriov_vf_map_complete_new(self):
utils.update_sriov_vf_map('eth1', 2, 'eth1_2', vlan_id=10, qos=5,
spoofcheck="on", trust="on", state="enable",

27
os_net_config/utils.py

@ -586,6 +586,32 @@ def _get_vf_name_from_map(pf_name, vfid):
return item['name']
def nicpart_udev_rules_check():
"""Clean-up UDEV rules on initial deployment
After writing sriov_config.yaml file, clean-up
UDEV rule(s) of PF for which VFs are used by host
"""
if not os.path.exists(sriov_config._UDEV_LEGACY_RULE_FILE):
return
udev = '^KERNEL=="(.*)", RUN.*'
udev_reg = re.compile(udev, 0)
with open(sriov_config._UDEV_LEGACY_RULE_FILE, "r") as fp:
rules = fp.readlines()
with open(sriov_config._UDEV_LEGACY_RULE_FILE, "w") as fp:
for line in rules:
match = udev_reg.match(line)
if match:
dev_name = match.group(1)
if not sriov_config.is_partitioned_pf(dev_name):
fp.write(line)
else:
fp.write(line)
def _configure_sriov_config_service():
"""Generate the sriov_config.service
@ -607,6 +633,7 @@ def configure_sriov_pfs(execution_from_cli=False, restart_openvswitch=False):
def configure_sriov_vfs():
logger.info("Configuring VFs now")
sriov_config.configure_sriov_vf()
nicpart_udev_rules_check()
def get_vf_devname(pf_name, vfid):

Loading…
Cancel
Save