Update sriov config service to handle nic partitioned PF
Create VFs using udev rules are required for attaching and removing PF from a VM. On a reboot, when udev rules trigger VF creation script and when a VF is attached to vfio-pci driver, some of the VFs on that PF are not initialized properly, resulting in VF not found issue. Though it looks like driver issue, this patch avoids this scenarios by removing the nic-paritioned VF from the udev rule, as when one of the VF is attached to host, the PF cannot be used by the VM. The function setUp_pf_stubs() was added in the vdpa patch: https://review.opendev.org/c/openstack/os-net-config/+/798926/ Since there is a dependency on this API for the tests of this patch, it has been added here Change-Id: Ic6738de42b9aa0346f48060c143c5f779b67867e
This commit is contained in:
parent
eee049618b
commit
62d4b0f2ac
|
@ -252,6 +252,24 @@ def udev_monitor_stop(observer):
|
|||
observer.stop()
|
||||
|
||||
|
||||
def is_partitioned_pf(dev_name):
|
||||
"""Check if any nic-partition(VF) is already used
|
||||
|
||||
Given a PF device, returns True if any VFs of this
|
||||
device are in-use.
|
||||
"""
|
||||
sriov_map = _get_sriov_map()
|
||||
for config in sriov_map:
|
||||
devtype = config.get('device_type', None)
|
||||
if devtype == 'vf':
|
||||
name = config.get('device', {}).get('name')
|
||||
vf_name = config.get('name')
|
||||
if dev_name == name:
|
||||
logger.warning("%s has VF(%s) used by host" % (name, vf_name))
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def configure_sriov_pf(execution_from_cli=False, restart_openvswitch=False):
|
||||
observer = udev_monitor_setup()
|
||||
udev_monitor_start(observer)
|
||||
|
@ -267,8 +285,9 @@ def configure_sriov_pf(execution_from_cli=False, restart_openvswitch=False):
|
|||
if item.get('link_mode') == "legacy":
|
||||
# Add a udev rule to configure the VF's when PF's are
|
||||
# released by a guest
|
||||
add_udev_rule_for_legacy_sriov_pf(item['name'],
|
||||
item['numvfs'])
|
||||
if not is_partitioned_pf(item['name']):
|
||||
add_udev_rule_for_legacy_sriov_pf(item['name'],
|
||||
item['numvfs'])
|
||||
set_numvfs(item['name'], item['numvfs'])
|
||||
vendor_id = get_vendor_id(item['name'])
|
||||
if (item.get('link_mode') == "switchdev" and
|
||||
|
@ -281,6 +300,7 @@ def configure_sriov_pf(execution_from_cli=False, restart_openvswitch=False):
|
|||
logger.info("Unbinding %s" % vf_pci)
|
||||
with open(MLNX_UNBIND_FILE_PATH, 'w') as f:
|
||||
f.write(vf_pci)
|
||||
|
||||
logger.info("%s: Adding udev rules" % item['name'])
|
||||
# Adding a udev rule to make vf-representors unmanaged by
|
||||
# NetworkManager
|
||||
|
@ -451,6 +471,7 @@ def configure_switchdev(pf_name):
|
|||
% (pf_pci, exc))
|
||||
raise
|
||||
logger.info("Device pci/%s set to switchdev mode." % pf_pci)
|
||||
|
||||
# WA to make sure that the uplink_rep is ready after moving to switchdev,
|
||||
# as moving to switchdev will remove the sriov_pf and create uplink
|
||||
# representor, so we need to make sure that uplink representor is ready
|
||||
|
@ -473,6 +494,7 @@ def configure_smfs_software_steering(pf_name):
|
|||
'pci/%s' % pf_pci, 'name', 'flow_steering_mode',
|
||||
'value', 'smfs', 'cmode', 'runtime')
|
||||
logger.info("Device pci/%s is set to smfs steering mode." % pf_pci)
|
||||
|
||||
except processutils.ProcessExecutionError as exc:
|
||||
logger.warning("Could not set pci/%s to smfs steering mode: %s"
|
||||
% (pf_pci, exc))
|
||||
|
|
|
@ -226,6 +226,52 @@ class TestSriovConfig(base.TestCase):
|
|||
f = open(udev_file.name, 'r')
|
||||
self.assertEqual(exp_udev_content, f.read())
|
||||
|
||||
def setUp_pf_stubs(self, vendor_id="0x8086"):
|
||||
run_cmd = []
|
||||
self.setUp_udev_stubs()
|
||||
|
||||
def run_ip_config_cmd_stub(*args, **kwargs):
|
||||
run_cmd.append(' '.join(args))
|
||||
self.stub_out('os_net_config.sriov_config.run_ip_config_cmd',
|
||||
run_ip_config_cmd_stub)
|
||||
|
||||
def cleanup_puppet_config_stub():
|
||||
return
|
||||
self.stub_out('os_net_config.sriov_config.cleanup_puppet_config',
|
||||
cleanup_puppet_config_stub)
|
||||
|
||||
def _wait_for_vf_creation_stub(pf_name, numvfs):
|
||||
self._save_action('_wait_for_vf_creation_stub')
|
||||
return
|
||||
self.stub_out('os_net_config.sriov_config._wait_for_vf_creation',
|
||||
_wait_for_vf_creation_stub)
|
||||
|
||||
def get_vendor_id_stub(ifname):
|
||||
return vendor_id
|
||||
self.stub_out('os_net_config.sriov_config.get_vendor_id',
|
||||
get_vendor_id_stub)
|
||||
|
||||
def configure_switchdev_stub(pf_name):
|
||||
self._save_action('configure_switchdev')
|
||||
return
|
||||
self.stub_out('os_net_config.sriov_config.configure_switchdev',
|
||||
configure_switchdev_stub)
|
||||
|
||||
self.set_numvfs = sriov_config.set_numvfs
|
||||
self.get_numvfs = sriov_config.get_numvfs
|
||||
|
||||
def get_numvfs_stub(*args):
|
||||
self._save_action('get_numvfs')
|
||||
return self.get_numvfs(*args)
|
||||
self.stub_out('os_net_config.sriov_config.get_numvfs',
|
||||
get_numvfs_stub)
|
||||
|
||||
def set_numvfs_stub(*args):
|
||||
self._save_action('set_numvfs')
|
||||
return self.set_numvfs(*args)
|
||||
self.stub_out('os_net_config.sriov_config.set_numvfs',
|
||||
set_numvfs_stub)
|
||||
|
||||
def test_configure_sriov_pf(self):
|
||||
"""Test the numvfs setting for SR-IOV PF
|
||||
|
||||
|
@ -273,6 +319,101 @@ class TestSriovConfig(base.TestCase):
|
|||
self.assertEqual(10, sriov_config.get_numvfs('p2p1'))
|
||||
self.assertEqual(12, sriov_config.get_numvfs('p2p2'))
|
||||
|
||||
def test_configure_sriov_pf_nicpart(self):
|
||||
"""Test the udev rules created for legacy mode of SR-IOV PF
|
||||
|
||||
In this case, the VF(s) are already bound/attached
|
||||
"""
|
||||
|
||||
self.setUp_pf_stubs()
|
||||
|
||||
exp_udev_content = '# This file is autogenerated by os-net-config\n'\
|
||||
'KERNEL=="p2p2", RUN+="/bin/os-net-config-sriov -n %k:12"\n'
|
||||
exp_actions = [
|
||||
'udev_monitor_setup',
|
||||
'udev_monitor_start',
|
||||
'set_numvfs',
|
||||
'get_numvfs',
|
||||
'_wait_for_vf_creation_stub',
|
||||
'get_numvfs',
|
||||
'reload_udev_rules',
|
||||
'set_numvfs',
|
||||
'get_numvfs',
|
||||
'_wait_for_vf_creation_stub',
|
||||
'get_numvfs',
|
||||
'udev_monitor_stop',
|
||||
]
|
||||
|
||||
pf_config = [{"device_type": "pf", "name": "p2p1", "numvfs": 10,
|
||||
"promisc": "on", "link_mode": "legacy"},
|
||||
{"device_type": "pf", "name": "p2p2", "numvfs": 12,
|
||||
"promisc": "off", "link_mode": "legacy"},
|
||||
{"device": {"name": "p2p1", "vfid": 0},
|
||||
"device_type": "vf", "name": "p2p1v0", "max_tx_rate": 0,
|
||||
"min_tx_rate": 0, "pci_address": "0000:18:0a.0",
|
||||
"trust": "on", "spoofcheck": "off"}]
|
||||
|
||||
for ifname in ['p2p1', 'p2p2']:
|
||||
self._write_numvfs(ifname)
|
||||
|
||||
self._action_order = []
|
||||
utils.write_yaml_config(sriov_config._SRIOV_CONFIG_FILE, pf_config)
|
||||
sriov_config.configure_logger(debug=True)
|
||||
sriov_config.configure_sriov_pf()
|
||||
self.assertEqual(exp_actions, self._action_order)
|
||||
f = open(sriov_config._UDEV_LEGACY_RULE_FILE, 'r')
|
||||
self.assertEqual(exp_udev_content, f.read())
|
||||
self.assertEqual(12, sriov_config.get_numvfs('p2p2'))
|
||||
|
||||
def test_configure_sriov_pf_non_nicpart(self):
|
||||
"""Test the udev rules created for legacy mode of SR-IOV PF
|
||||
|
||||
In this case, the nic partitioned VF(s) are not attached
|
||||
"""
|
||||
|
||||
self.setUp_pf_stubs()
|
||||
|
||||
exp_udev_content = '# This file is autogenerated by os-net-config\n'\
|
||||
'KERNEL=="p2p1", RUN+="/bin/os-net-config-sriov -n %k:10"\n'\
|
||||
'KERNEL=="p2p2", RUN+="/bin/os-net-config-sriov -n %k:12"\n'
|
||||
exp_actions = [
|
||||
'udev_monitor_setup',
|
||||
'udev_monitor_start',
|
||||
'reload_udev_rules',
|
||||
'set_numvfs',
|
||||
'get_numvfs',
|
||||
'_wait_for_vf_creation_stub',
|
||||
'get_numvfs',
|
||||
'reload_udev_rules',
|
||||
'set_numvfs',
|
||||
'get_numvfs',
|
||||
'_wait_for_vf_creation_stub',
|
||||
'get_numvfs',
|
||||
'udev_monitor_stop',
|
||||
]
|
||||
|
||||
pf_config = [{"device_type": "pf", "name": "p2p1", "numvfs": 10,
|
||||
"promisc": "on", "link_mode": "legacy"},
|
||||
{"device_type": "pf", "name": "p2p2", "numvfs": 12,
|
||||
"promisc": "off", "link_mode": "legacy"},
|
||||
{"device": {"name": "eno3", "vfid": 1},
|
||||
"device_type": "vf", "name": "eno3v0", "max_tx_rate": 0,
|
||||
"min_tx_rate": 0, "pci_address": "0000:18:0e.1",
|
||||
"trust": "on", "spoofcheck": "off"}]
|
||||
|
||||
for ifname in ['p2p1', 'p2p2']:
|
||||
self._write_numvfs(ifname)
|
||||
|
||||
self._action_order = []
|
||||
utils.write_yaml_config(sriov_config._SRIOV_CONFIG_FILE, pf_config)
|
||||
sriov_config.configure_logger(debug=True)
|
||||
sriov_config.configure_sriov_pf()
|
||||
self.assertEqual(exp_actions, self._action_order)
|
||||
f = open(sriov_config._UDEV_LEGACY_RULE_FILE, 'r')
|
||||
self.assertEqual(exp_udev_content, f.read())
|
||||
self.assertEqual(10, sriov_config.get_numvfs('p2p1'))
|
||||
self.assertEqual(12, sriov_config.get_numvfs('p2p2'))
|
||||
|
||||
def test_cleanup_puppet_config_deprecation(self):
|
||||
"""Test the cleanup of puppet-tripleo generated config file.
|
||||
|
||||
|
|
|
@ -75,6 +75,11 @@ dpdk {
|
|||
}
|
||||
'''
|
||||
|
||||
udev_content = '# This file is autogenerated by os-net-config\n'\
|
||||
'KERNEL=="eth1", RUN+="/bin/os-net-config-sriov -n %k:10"\n' \
|
||||
'KERNEL=="eth2", RUN+="/bin/os-net-config-sriov -n %k:10"\n'
|
||||
UDEV_FILE = '/tmp/70-os-net-config-sriov.'
|
||||
|
||||
|
||||
class TestUtils(base.TestCase):
|
||||
|
||||
|
@ -83,6 +88,7 @@ class TestUtils(base.TestCase):
|
|||
rand = str(int(random.random() * 100000))
|
||||
utils._DPDK_MAPPING_FILE = '/tmp/dpdk_mapping_' + rand + '.yaml'
|
||||
sriov_config._SRIOV_CONFIG_FILE = '/tmp/sriov_config_' + rand + '.yaml'
|
||||
sriov_config._UDEV_LEGACY_RULE_FILE = UDEV_FILE + rand
|
||||
|
||||
def tearDown(self):
|
||||
super(TestUtils, self).tearDown()
|
||||
|
@ -90,6 +96,8 @@ class TestUtils(base.TestCase):
|
|||
os.remove(utils._DPDK_MAPPING_FILE)
|
||||
if os.path.isfile(sriov_config._SRIOV_CONFIG_FILE):
|
||||
os.remove(sriov_config._SRIOV_CONFIG_FILE)
|
||||
if os.path.isfile(sriov_config._UDEV_LEGACY_RULE_FILE):
|
||||
os.remove(sriov_config._UDEV_LEGACY_RULE_FILE)
|
||||
|
||||
def test_ordered_active_nics(self):
|
||||
|
||||
|
@ -204,6 +212,37 @@ class TestUtils(base.TestCase):
|
|||
'device': {"name": "eth1", "vfid": 2}}]
|
||||
self.assertListEqual(test_sriov_vf_map, sriov_vf_map)
|
||||
|
||||
def test_udev_rule_for_sriov_vf(self):
|
||||
def get_numvfs_stub(pf_name):
|
||||
return 10
|
||||
self.stub_out('os_net_config.sriov_config.get_numvfs',
|
||||
get_numvfs_stub)
|
||||
utils.update_sriov_pf_map('eth1', 10, False)
|
||||
utils.update_sriov_pf_map('eth2', 10, False)
|
||||
utils.update_sriov_vf_map('eth1', 2, 'eth1_2')
|
||||
contents = utils.get_file_data(sriov_config._SRIOV_CONFIG_FILE)
|
||||
sriov_vf_map = yaml.safe_load(contents) if contents else []
|
||||
self.assertEqual(3, len(sriov_vf_map))
|
||||
|
||||
test_sriov_vf_map = {'device': {'name': 'eth1', 'vfid': 2},
|
||||
'device_type': 'vf', 'max_tx_rate': 0,
|
||||
'min_tx_rate': 0, 'name': 'eth1_2'}
|
||||
udev_file = open(sriov_config._UDEV_LEGACY_RULE_FILE, "w")
|
||||
udev_file.write(udev_content)
|
||||
udev_file.close()
|
||||
|
||||
# sriov_config.add_udev_rule_for_legacy_sriov_pf("eth1", 10)
|
||||
for dev in sriov_vf_map:
|
||||
if(dev.keys() == test_sriov_vf_map.keys()):
|
||||
self.assertEqual(test_sriov_vf_map, dev)
|
||||
|
||||
expect = '# This file is autogenerated by os-net-config\n'\
|
||||
'KERNEL=="eth2", RUN+="/bin/os-net-config-sriov -n %k:10"\n'
|
||||
utils.nicpart_udev_rules_check()
|
||||
f = open(sriov_config._UDEV_LEGACY_RULE_FILE, 'r')
|
||||
self.assertEqual(expect, f.read())
|
||||
f.close()
|
||||
|
||||
def test_update_sriov_vf_map_complete_new(self):
|
||||
utils.update_sriov_vf_map('eth1', 2, 'eth1_2', vlan_id=10, qos=5,
|
||||
spoofcheck="on", trust="on", state="enable",
|
||||
|
|
|
@ -586,6 +586,32 @@ def _get_vf_name_from_map(pf_name, vfid):
|
|||
return item['name']
|
||||
|
||||
|
||||
def nicpart_udev_rules_check():
|
||||
"""Clean-up UDEV rules on initial deployment
|
||||
|
||||
After writing sriov_config.yaml file, clean-up
|
||||
UDEV rule(s) of PF for which VFs are used by host
|
||||
"""
|
||||
if not os.path.exists(sriov_config._UDEV_LEGACY_RULE_FILE):
|
||||
return
|
||||
|
||||
udev = '^KERNEL=="(.*)", RUN.*'
|
||||
udev_reg = re.compile(udev, 0)
|
||||
|
||||
with open(sriov_config._UDEV_LEGACY_RULE_FILE, "r") as fp:
|
||||
rules = fp.readlines()
|
||||
|
||||
with open(sriov_config._UDEV_LEGACY_RULE_FILE, "w") as fp:
|
||||
for line in rules:
|
||||
match = udev_reg.match(line)
|
||||
if match:
|
||||
dev_name = match.group(1)
|
||||
if not sriov_config.is_partitioned_pf(dev_name):
|
||||
fp.write(line)
|
||||
else:
|
||||
fp.write(line)
|
||||
|
||||
|
||||
def _configure_sriov_config_service():
|
||||
"""Generate the sriov_config.service
|
||||
|
||||
|
@ -607,6 +633,7 @@ def configure_sriov_pfs(execution_from_cli=False, restart_openvswitch=False):
|
|||
def configure_sriov_vfs():
|
||||
logger.info("Configuring VFs now")
|
||||
sriov_config.configure_sriov_vf()
|
||||
nicpart_udev_rules_check()
|
||||
|
||||
|
||||
def get_vf_devname(pf_name, vfid):
|
||||
|
|
Loading…
Reference in New Issue