openstack
/
os-net-config
Code Issues Proposed changes

vdpa: Introducing the vdpa PF flag 

Now that nova supports vDPA interfaces, we need to configure them using
os_net_config for tripleo consumption.

Related: https://review.opendev.org/c/openstack/tripleo-docs/+/798237/
Related: https://bugzilla.redhat.com/1956297
Change-Id: I202fe57360cca79883457444bdae2cf41ba25798
This commit is contained in:
David Vallee Delisle 2021-06-30 12:03:30 -04:00
parent 380254e897
commit baab05e9bb
11 changed files with 256 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
etc/os-net-config/samples/vdpa.json Normal file
View File

@ -0,0 +1,28 @@
{
"network_config": [
{
"type": "ovs_bridge",
"name": "br-vdpa",
"members": [
{
"type": "sriov_pf",
"name": "p2p1",
"numvfs": 10,
"vdpa": true,
"use_dhcp": false,
"link_mode": "switchdev"
},
{
"type": "sriov_pf",
"name": "p2p2",
"numvfs": 10,
"vdpa": true,
"use_dhcp": false,
"link_mode": "switchdev"
}
],
"use_dhcp": true
}
]
}

31
etc/os-net-config/samples/vdpa.yaml Normal file
View File

@ -0,0 +1,31 @@
network_config:
# Since vDPA can't function without OVS Hardware Offload, the PFs have to
# be a member of an ovs_bridge.
- type: ovs_bridge
name: br-vdpa
use_dhcp: true
members:
# sriov_pf type shall be used to configure the PF's of vDPA devices.
# The numvfs configured for the PF's shall be set on the sriov_numvfs of the
# sysfs for the corresponding NIC and the persistence of the same across reboots
# shall be handled
-
type: sriov_pf
# nic name or nic number of the NIC that needs to be configured for vDPA
name: p2p1
# Flag this PF as a vDPA device
vdpa: true
# number of VFs required on the particular NIC
# Should be at least one when vdpa is true
numvfs: 10
# Dont set the IP address on the PF
use_dhcp: false
# Link mode for the PF has to be switchdev when vdpa is true
link_mode: switchdev
-
type: sriov_pf
name: p2p2
vdpa: true
numvfs: 10
use_dhcp: false
link_mode: switchdev

3
os_net_config/impl_ifcfg.py
View File

@ -1013,7 +1013,8 @@ class IfcfgNetConfig(os_net_config.NetConfig):
logger.debug('sriov pf data: %s' % data)
utils.update_sriov_pf_map(sriov_pf.name, sriov_pf.numvfs,
self.noop, promisc=sriov_pf.promisc,
link_mode=sriov_pf.link_mode)
link_mode=sriov_pf.link_mode,
vdpa=sriov_pf.vdpa)
self.interface_data[sriov_pf.name] = data
if sriov_pf.routes:
self._add_routes(sriov_pf.name, sriov_pf.routes)

18
os_net_config/objects.py
View File

@ -1515,7 +1515,8 @@ class SriovPF(_BaseOpts):
primary=False, nic_mapping=None, persist_mapping=False,
defroute=True, dhclient_args=None, dns_servers=None,
nm_controlled=False, onboot=True, domain=None, members=None,
promisc=None, link_mode='legacy', ethtool_opts=None):
promisc=None, link_mode='legacy', ethtool_opts=None,
vdpa=False):
addresses = addresses or []
routes = routes or []
rules = rules or []
@ -1534,6 +1535,7 @@ class SriovPF(_BaseOpts):
self.promisc = promisc
self.link_mode = link_mode
self.ethtool_opts = ethtool_opts
self.vdpa = vdpa
@staticmethod
def get_on_off(config):
@ -1553,12 +1555,24 @@ class SriovPF(_BaseOpts):
promisc = SriovPF.get_on_off(promisc)
link_mode = json.get('link_mode', 'legacy')
ethtool_opts = json.get('ethtool_opts', None)
vdpa = json.get('vdpa', False)
if vdpa:
msg = ""
if link_mode != 'switchdev':
msg += "Expecting link_mode to be switchdev when vdpa is "\
f"enabled, not {link_mode}\n"
if not int(numvfs):
msg += ("Expecting to have at least 1 numvfs when vdpa is "
"enabled\n")
if len(msg):
raise InvalidConfigException(msg)
if link_mode not in ['legacy', 'switchdev']:
msg = 'Expecting link_mode to match legacy/switchdev'
raise InvalidConfigException(msg)
opts = _BaseOpts.base_opts_from_json(json)
return SriovPF(name, numvfs, *opts, promisc=promisc,
link_mode=link_mode, ethtool_opts=ethtool_opts)
link_mode=link_mode, ethtool_opts=ethtool_opts,
vdpa=vdpa)
class OvsDpdkBond(_BaseOpts):

2
os_net_config/schema.yaml
View File

@ -363,6 +363,8 @@ definitions:
$ref: "#/definitions/list_of_domain_name_string_or_domain_name_string"
link_mode:
$ref: "#/definitions/sriov_link_mode_or_param"
vdpa:
$ref: "#/definitions/bool_or_param"
required:
- type
- name

18
os_net_config/sriov_config.py
View File

@ -43,6 +43,7 @@ _RESET_SRIOV_RULES_FILE = '/etc/udev/rules.d/70-tripleo-reset-sriov.rules'
_ALLOCATE_VFS_FILE = '/etc/sysconfig/allocate_vfs'
_MLNX_DRIVER = "mlx5_core"
MLNX_VENDOR_ID = "0x15b3"
MLNX_UNBIND_FILE_PATH = "/sys/bus/pci/drivers/mlx5_core/unbind"
MAX_RETRIES = 10
PF_FUNC_RE = re.compile(r"\.(\d+)$", 0)
@ -199,7 +200,8 @@ def set_numvfs(ifname, numvfs):
sriov_numvfs_path = _get_dev_path(ifname, "sriov_numvfs")
try:
with open(sriov_numvfs_path, 'w') as f:
logger.debug(f"Setting {sriov_numvfs_path} to {numvfs}")
with open(sriov_numvfs_path, "w") as f:
f.write("%d" % numvfs)
except IOError as exc:
msg = (f"Unable to configure pf: {ifname} with numvfs: {numvfs}\n"
@ -268,7 +270,6 @@ def configure_sriov_pf(execution_from_cli=False, restart_openvswitch=False):
udev_monitor_start(observer)
sriov_map = _get_sriov_map()
MLNX_UNBIND_FILE_PATH = "/sys/bus/pci/drivers/mlx5_core/unbind"
mlnx_vfs_pcis_list = []
trigger_udev_rule = False
@ -283,10 +284,17 @@ def configure_sriov_pf(execution_from_cli=False, restart_openvswitch=False):
# released by a guest
add_udev_rule_for_legacy_sriov_pf(item['name'],
item['numvfs'])
set_numvfs(item['name'], item['numvfs'])
# When configuring vdpa, we need to configure switchdev before
# we create the VFs
vendor_id = get_vendor_id(item['name'])
if (item.get('link_mode') == "switchdev" and
vendor_id == MLNX_VENDOR_ID):
is_mlnx = vendor_id == MLNX_VENDOR_ID
# Configure switchdev mode when vdpa
if item.get('vdpa') and is_mlnx:
configure_switchdev(item['name'])
set_numvfs(item['name'], item['numvfs'])
# Configure switchdev mode when not vdpa
if (item.get('link_mode') == "switchdev" and is_mlnx and
not item.get('vdpa')):
logger.info(f"{item['name']}: Mellanox card")
vf_pcis_list = get_vf_pcis_list(item['name'])
mlnx_vfs_pcis_list += vf_pcis_list

6
os_net_config/tests/test_impl_ifcfg.py
View File

@ -1615,7 +1615,7 @@ NETMASK=255.255.255.0
pf = objects.SriovPF(name='nic3', numvfs=10)
def test_update_sriov_pf_map(name, numvfs, noop, promisc=None,
link_mode='legacy'):
link_mode='legacy', vdpa=False):
self.assertEqual(name, 'eth2')
self.assertEqual(numvfs, 10)
self.assertEqual(promisc, None)
@ -1640,7 +1640,7 @@ BOOTPROTO=none
pf = objects.SriovPF(name='nic3', numvfs=10, promisc=True)
def test_update_sriov_pf_map(name, numvfs, noop, promisc=None,
link_mode='legacy'):
link_mode='legacy', vdpa=False):
self.assertEqual(name, 'eth2')
self.assertEqual(numvfs, 10)
self.assertTrue(promisc)
@ -1665,7 +1665,7 @@ BOOTPROTO=none
pf = objects.SriovPF(name='nic3', numvfs=10, promisc=False)
def test_update_sriov_pf_map(name, numvfs, noop, promisc=None,
link_mode='legacy'):
link_mode='legacy', vdpa=False):
self.assertEqual(name, 'eth2')
self.assertEqual(numvfs, 10)
self.assertFalse(promisc)

20
os_net_config/tests/test_objects.py
View File

@ -1984,6 +1984,26 @@ class TestSriovPF(base.TestCase):
expected = 'Expecting link_mode to match legacy/switchdev'
self.assertIn(expected, six.text_type(err))
def test_from_json_vdpa_link_mode_invalid(self):
data = '{"type": "sriov_pf", "name": "p6p1", "numvfs": 8,' \
'"use_dhcp": false, "promisc": false, "link_mode":' \
'"none_switchdev", "vdpa": true}'
err = self.assertRaises(objects.InvalidConfigException,
objects.object_from_json,
json.loads(data))
expected = 'Expecting link_mode to be switchdev when vdpa is enabled'
self.assertIn(expected, six.text_type(err))
def test_from_json_vdpa_no_numvfs(self):
data = '{"type": "sriov_pf", "name": "p6p1", "numvfs": 0,' \
'"use_dhcp": false, "promisc": false, "link_mode":' \
'"switchdev", "vdpa": true}'
err = self.assertRaises(objects.InvalidConfigException,
objects.object_from_json,
json.loads(data))
expected = 'Expecting to have at least 1 numvfs when vdpa is enabled'
self.assertIn(expected, six.text_type(err))
def test_from_json_ethtool_opts(self):
data = '{"type": "sriov_pf", "name": "em1", "numvfs": 16, ' \
'"use_dhcp": false, "promisc": false, ' \

110
os_net_config/tests/test_sriov_config.py
View File

@ -226,15 +226,7 @@ class TestSriovConfig(base.TestCase):
f = open(udev_file.name, 'r')
self.assertEqual(exp_udev_content, f.read())
def test_configure_sriov_pf(self):
"""Test the numvfs setting for SR-IOV PF
Test the udev rules created for legacy mode of SR-IOV PF
"""
exp_udev_content = '# This file is autogenerated by os-net-config\n'\
'KERNEL=="p2p1", RUN+="/bin/os-net-config-sriov -n %k:10"\n'\
'KERNEL=="p2p2", RUN+="/bin/os-net-config-sriov -n %k:12"\n'
def setUp_pf_stubs(self, vendor_id="0x8086"):
run_cmd = []
@ -245,17 +237,70 @@ class TestSriovConfig(base.TestCase):
self.stub_out('os_net_config.sriov_config.run_ip_config_cmd',
run_ip_config_cmd_stub)
def cleanup_puppet_config_stub():
return
self.stub_out('os_net_config.sriov_config.cleanup_puppet_config',
cleanup_puppet_config_stub)
def _wait_for_vf_creation_stub(pf_name, numvfs):
numvfs_pair = {"p2p1": 10, "p2p2": 12}
self.assertEqual(numvfs_pair[pf_name], numvfs)
self._save_action('_wait_for_vf_creation_stub')
return
self.stub_out('os_net_config.sriov_config._wait_for_vf_creation',
_wait_for_vf_creation_stub)
def get_vendor_id_stub(ifname):
return "0x8086"
return vendor_id
self.stub_out('os_net_config.sriov_config.get_vendor_id',
get_vendor_id_stub)
def configure_switchdev_stub(pf_name):
self._save_action('configure_switchdev')
return
self.stub_out('os_net_config.sriov_config.configure_switchdev',
configure_switchdev_stub)
self.set_numvfs = sriov_config.set_numvfs
self.get_numvfs = sriov_config.get_numvfs
def get_numvfs_stub(*args):
self._save_action('get_numvfs')
return self.get_numvfs(*args)
self.stub_out('os_net_config.sriov_config.get_numvfs',
get_numvfs_stub)
def set_numvfs_stub(*args):
self._save_action('set_numvfs')
return self.set_numvfs(*args)
self.stub_out('os_net_config.sriov_config.set_numvfs',
set_numvfs_stub)
def test_configure_sriov_pf(self):
"""Test the numvfs setting for SR-IOV PF
Test the udev rules created for legacy mode of SR-IOV PF
"""
self.setUp_pf_stubs()
exp_udev_content = '# This file is autogenerated by os-net-config\n'\
'KERNEL=="p2p1", RUN+="/bin/os-net-config-sriov -n %k:10"\n'\
'KERNEL=="p2p2", RUN+="/bin/os-net-config-sriov -n %k:12"\n'
exp_actions = [
'udev_monitor_setup',
'udev_monitor_start',
'reload_udev_rules',
'set_numvfs',
'get_numvfs',
'_wait_for_vf_creation_stub',
'get_numvfs',
'reload_udev_rules',
'set_numvfs',
'get_numvfs',
'_wait_for_vf_creation_stub',
'get_numvfs',
'udev_monitor_stop',
]
pf_config = [{"device_type": "pf", "name": "p2p1", "numvfs": 10,
"promisc": "on", "link_mode": "legacy"},
{"device_type": "pf", "name": "p2p2", "numvfs": 12,
@ -264,15 +309,54 @@ class TestSriovConfig(base.TestCase):
for ifname in ['p2p1', 'p2p2']:
self._write_numvfs(ifname)
self._action_order = []
utils.write_yaml_config(sriov_config._SRIOV_CONFIG_FILE, pf_config)
sriov_config.configure_logger(debug=True)
sriov_config.configure_sriov_pf()
self.assertEqual(exp_actions, self._action_order)
f = open(sriov_config._UDEV_LEGACY_RULE_FILE, 'r')
self.assertEqual(exp_udev_content, f.read())
self.assertEqual(10, sriov_config.get_numvfs('p2p1'))
self.assertEqual(12, sriov_config.get_numvfs('p2p2'))
def test_configure_vdpa_pf(self):
"""Test the vdpa PF
"""
self.setUp_pf_stubs(sriov_config.MLNX_VENDOR_ID)
exp_actions = [
'udev_monitor_setup',
'udev_monitor_start',
'configure_switchdev',
'set_numvfs',
'get_numvfs',
'_wait_for_vf_creation_stub',
'get_numvfs',
'configure_switchdev',
'set_numvfs',
'get_numvfs',
'_wait_for_vf_creation_stub',
'get_numvfs',
'udev_monitor_stop',
]
pf_config = [{"device_type": "pf", "name": "p2p1", "numvfs": 10,
"vdpa": True, "link_mode": "switchdev"},
{"device_type": "pf", "name": "p2p2", "numvfs": 12,
"vdpa": True, "link_mode": "switchdev"}]
for ifname in ['p2p1', 'p2p2']:
self._write_numvfs(ifname)
self._action_order = []
utils.write_yaml_config(sriov_config._SRIOV_CONFIG_FILE, pf_config)
sriov_config.configure_logger(debug=True)
sriov_config.configure_sriov_pf()
self.assertEqual(exp_actions, self._action_order)
self.assertEqual(10, sriov_config.get_numvfs('p2p1'))
self.assertEqual(12, sriov_config.get_numvfs('p2p2'))
def test_cleanup_puppet_config_deprecation(self):
"""Test the cleanup of puppet-tripleo generated config file.

46
os_net_config/tests/test_utils.py
View File

@ -128,7 +128,7 @@ class TestUtils(base.TestCase):
sriov_pf_map = yaml.safe_load(contents) if contents else []
self.assertEqual(1, len(sriov_pf_map))
test_sriov_pf_map = [{'device_type': 'pf', 'link_mode': 'legacy',
'name': 'eth1', 'numvfs': 10}]
'name': 'eth1', 'numvfs': 10, 'vdpa': False}]
self.assertListEqual(test_sriov_pf_map, sriov_pf_map)
def test_update_sriov_pf_map_with_same_numvfs(self):
@ -141,7 +141,7 @@ class TestUtils(base.TestCase):
sriov_pf_map = yaml.safe_load(contents) if contents else []
self.assertEqual(1, len(sriov_pf_map))
test_sriov_pf_map = [{'device_type': 'pf', 'link_mode': 'legacy',
'name': 'eth1', 'numvfs': 10}]
'name': 'eth1', 'numvfs': 10, 'vdpa': False}]
self.assertListEqual(test_sriov_pf_map, sriov_pf_map)
def test_update_sriov_pf_map_with_diff_numvfs(self):
@ -162,7 +162,21 @@ class TestUtils(base.TestCase):
sriov_pf_map = yaml.safe_load(contents) if contents else []
self.assertEqual(1, len(sriov_pf_map))
test_sriov_pf_map = [{'device_type': 'pf', 'link_mode': 'legacy',
'name': 'eth1', 'numvfs': 10, 'promisc': 'off'}]
'name': 'eth1', 'numvfs': 10, 'promisc': 'off',
'vdpa': False}]
self.assertListEqual(test_sriov_pf_map, sriov_pf_map)
def test_update_sriov_pf_map_new_with_vdpa(self):
def get_numvfs_stub(pf_name):
return 0
self.stub_out('os_net_config.sriov_config.get_numvfs',
get_numvfs_stub)
utils.update_sriov_pf_map('eth1', 10, False, vdpa=True)
contents = utils.get_file_data(sriov_config._SRIOV_CONFIG_FILE)
sriov_pf_map = yaml.safe_load(contents) if contents else []
self.assertEqual(1, len(sriov_pf_map))
test_sriov_pf_map = [{'device_type': 'pf', 'link_mode': 'legacy',
'name': 'eth1', 'numvfs': 10, 'vdpa': True}]
self.assertListEqual(test_sriov_pf_map, sriov_pf_map)
def test_update_sriov_pf_map_exist(self):
@ -182,12 +196,34 @@ class TestUtils(base.TestCase):
self.stub_out('os_net_config.sriov_config.get_numvfs',
get_numvfs_stub)
pf_initial = [{'device_type': 'pf', 'link_mode': 'legacy',
'name': 'eth1', 'numvfs': 10, 'promisc': 'on'}]
'name': 'eth1', 'numvfs': 10, 'promisc': 'on',
'vdpa': False}]
utils.write_yaml_config(sriov_config._SRIOV_CONFIG_FILE, pf_initial)
utils.update_sriov_pf_map('eth1', 10, False, promisc='off')
pf_final = [{'device_type': 'pf', 'link_mode': 'legacy',
'name': 'eth1', 'numvfs': 10, 'promisc': 'off'}]
'name': 'eth1', 'numvfs': 10, 'promisc': 'off',
'vdpa': False}]
contents = utils.get_file_data(sriov_config._SRIOV_CONFIG_FILE)
pf_map = yaml.safe_load(contents) if contents else []
self.assertEqual(1, len(pf_map))
self.assertListEqual(pf_final, pf_map)
def test_update_sriov_pf_map_exist_with_vdpa(self):
def get_numvfs_stub(pf_name):
return 10
self.stub_out('os_net_config.sriov_config.get_numvfs',
get_numvfs_stub)
pf_initial = [{'device_type': 'pf', 'link_mode': 'legacy',
'name': 'eth1', 'numvfs': 10, 'promisc': 'on',
'vdpa': False}]
utils.write_yaml_config(sriov_config._SRIOV_CONFIG_FILE, pf_initial)
utils.update_sriov_pf_map('eth1', 10, False, vdpa=True)
pf_final = [{'device_type': 'pf', 'link_mode': 'legacy',
'name': 'eth1', 'numvfs': 10, 'promisc': 'on',
'vdpa': True}]
contents = utils.get_file_data(sriov_config._SRIOV_CONFIG_FILE)
pf_map = yaml.safe_load(contents) if contents else []

4
os_net_config/utils.py
View File

@ -488,7 +488,7 @@ def _get_dpdk_mac_address(name):
def update_sriov_pf_map(ifname, numvfs, noop, promisc=None,
link_mode='legacy'):
link_mode='legacy', vdpa=False):
if not noop:
cur_numvfs = sriov_config.get_numvfs(ifname)
if cur_numvfs > 0 and cur_numvfs != numvfs:
@ -498,6 +498,7 @@ def update_sriov_pf_map(ifname, numvfs, noop, promisc=None,
for item in sriov_map:
if item['device_type'] == 'pf' and item['name'] == ifname:
item['numvfs'] = numvfs
item['vdpa'] = vdpa
if promisc is not None:
item['promisc'] = promisc
item['link_mode'] = link_mode
@ -507,6 +508,7 @@ def update_sriov_pf_map(ifname, numvfs, noop, promisc=None,
new_item['device_type'] = 'pf'
new_item['name'] = ifname
new_item['numvfs'] = numvfs
new_item['vdpa'] = vdpa
if promisc is not None:
new_item['promisc'] = promisc
new_item['link_mode'] = link_mode