openstack/puppet-tripleo
Code Issues Proposed changes

Adding support for VF LAG in SR-IOV for Mellanox interfaces

Browse Source 
It is possible to configure bond over two virtual functions
for the vms in case of using mellanox interfaces.

Change-Id: Iaeee31a9edaefec25498a734cac6eda389c38ec5
This commit is contained in:
waleed mousa 2018-04-03 03:05:55 -04:00
parent 864f501b53
commit cd7232bb8c
4 changed files with 134 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

96
lib/puppet/provider/sriov_vf_config/numvfs.rb
View File

@ -10,8 +10,11 @@ Puppet::Type.type(:sriov_vf_config).provide(:numvfs) do
def create
if File.file?(sriov_numvfs_path)
_set_numvfs
_apply_hw_offload
if ovs_mode == "switchdev"
_apply_hw_offload
else
_set_numvfs
end
else
warning("#{sriov_numvfs_path} doesn't exist. Check if #{sriov_get_interface} is a valid network interface supporting SR-IOV")
end
@ -45,25 +48,53 @@ Puppet::Type.type(:sriov_vf_config).provide(:numvfs) do
def _apply_hw_offload
# Changing the mode of virtual functions to hw-offload
if ovs_mode == "switchdev"
cur_value = File.read(vendor_path).strip
if cur_value == "0x15b3"
vfs_pcis = get_vfs_pcis
# Unbinding virtual functions
vfs_pcis.each do|vfs_pci|
File.write("/sys/bus/pci/drivers/mlx5_core/unbind",vfs_pci)
bond_enabled = get_bond_enabled
vendor_id = File.read(vendor_path).strip
# Adding the VF LAG
if vendor_id == "0x15b3" and bond_enabled
bond_masters = get_bond_masters
bond_master_hash = {}
bond_masters.each do |bond|
bond_interfaces = get_bond_interfaces(bond)
bond_master_hash[bond] = bond_interfaces
end
# Removing the slaves from the bond interfaces
bond_master_hash.each do |bond, bond_interfaces|
bond_interfaces.each do |bond_interface|
%x{echo "-#{bond_interface}" > /sys/class/net/#{bond}/bonding/slaves}
end
end
# Changing the mode of sriov interface to switchdev mode
%x{/usr/sbin/devlink dev eswitch set pci/#{get_interface_pci} mode switchdev}
if get_interface_device == "0x1013" || get_interface_device == "0x1015"
%x{/usr/sbin/devlink dev eswitch set pci/#{get_interface_pci} inline-mode transport}
end
# Setting the number of vfs
_set_numvfs
# Applying the hardware offloading
if vendor_id == "0x15b3"
vfs_pcis = get_vfs_pcis
# Unbinding virtual functions
vfs_pcis.each do|vfs_pci|
File.write("/sys/bus/pci/drivers/mlx5_core/unbind",vfs_pci)
end
%x{/usr/sbin/ethtool -K #{sriov_get_interface} hw-tc-offload on}
if cur_value == "0x15b3"
# Binding virtual functions
vfs_pcis.each do|vfs_pci|
File.write("/sys/bus/pci/drivers/mlx5_core/bind",vfs_pci)
end
# Changing the mode of sriov interface to switchdev mode
%x{/usr/sbin/devlink dev eswitch set pci/#{get_interface_pci} mode switchdev}
if get_interface_device == "0x1013" || get_interface_device == "0x1015"
%x{/usr/sbin/devlink dev eswitch set pci/#{get_interface_pci} inline-mode transport}
end
%x{/usr/sbin/ethtool -K #{sriov_get_interface} hw-tc-offload on}
if vendor_id == "0x15b3"
# Binding virtual functions
vfs_pcis.each do|vfs_pci|
File.write("/sys/bus/pci/drivers/mlx5_core/bind",vfs_pci)
end
end
if vendor_id and bond_enabled
# Adding the slaves back to the bond interfaces
bond_master_hash.each do |bond, bond_interfaces|
bond_interfaces.each do |bond_interface|
%x{echo "+#{bond_interface}" > /sys/class/net/#{bond}/bonding/slaves}
end
end
end
@ -104,4 +135,33 @@ Puppet::Type.type(:sriov_vf_config).provide(:numvfs) do
def get_interface_device
%x{cat /sys/class/net/#{sriov_get_interface}/device/device}.strip
end
def get_bond_enabled
if %x{lsmod | grep bonding}.strip.length > 0
true
else
false
end
end
def bond_masters_path
"/sys/class/net/bonding_masters"
end
def get_bond_masters
if File.file?(bond_masters_path)
File.read(bond_masters_path).split()
end
end
def get_bond_interfaces_path(bond)
"/sys/class/net/#{bond}/bonding/slaves"
end
def get_bond_interfaces(bond)
if File.file?(get_bond_interfaces_path(bond))
File.read(get_bond_interfaces_path(bond)).split()
end
end
end

4
releasenotes/notes/vf-lag-sriov-ec194ecd4b447a46.yaml Normal file
View File

@ -0,0 +1,4 @@
---
features:
- |
Allows to configure bond over two virtual functions in mellanox interfaces.

29
spec/defines/tripleo_host_sriov_numvfs_persistence_spec.rb
View File

@ -68,19 +68,32 @@ describe 'tripleo::host::sriov::numvfs_persistence' do
it 'configures persistence' do
is_expected.to contain_file('/etc/sysconfig/allocate_vfs').with(
:ensure => 'file',
:content => "Hashbang\nset -eux
:content => "Hashbang\nset -ex
set -o pipefail
if [ \"eth0\" == \"$1\" ]
then
exec 1> >(logger -s -t $(basename $0)) 2>&1
vendor_id=\"$(cat /sys/class/net/eth0/device/vendor)\"
if [[ $vendor_id == \"0x15b3\" && $(/sbin/lsmod | grep bonding) ]]
then
declare -A bonds
for bond in `cat /sys/class/net/bonding_masters`
do
bonds[$bond]=`cat \"/sys/class/net/$bond/bonding/slaves\"`
for bond_interface in ${bonds[$bond]}
do
echo \"-$bond_interface\" > \"/sys/class/net/$bond/bonding/slaves\"
done
done
fi
if [ \"$(cat /sys/class/net/eth0/device/sriov_numvfs)\" == \"0\" ]
then
echo 10 > /sys/class/net/eth0/device/sriov_numvfs
else
exit 0
fi
if [ \"$(cat /sys/class/net/eth0/device/vendor)\" == \"0x15b3\" ]
if [ $vendor_id == \"0x15b3\" ]
then
vfs_pci_list=$(grep PCI_SLOT_NAME /sys/class/net/eth0/device/virtfn*/uevent | cut -d'=' -f2)
for pci in $vfs_pci_list
@ -103,13 +116,23 @@ then
/usr/sbin/devlink dev eswitch set pci/\"$interface_pci\" inline-mode transport
fi
/usr/sbin/ethtool -K eth0 hw-tc-offload on
if [ \"$(cat /sys/class/net/eth0/device/vendor)\" == \"0x15b3\" ]
if [ $vendor_id == \"0x15b3\" ]
then
for pci in $vfs_pci_list
do
echo \"$pci\" > /sys/bus/pci/drivers/mlx5_core/bind
done
fi
if [[ $vendor_id == \"0x15b3\" && $(/sbin/lsmod | grep bonding) ]]
then
for bond in \"${!bonds[@]}\"
do
for bond_interface in ${bonds[$bond]}
do
echo \"+$bond_interface\" > \"/sys/class/net/$bond/bonding/slaves\"
done
done
fi
fi\n[ \"eth1\" == \"\$1\" ] && echo 8 > /sys/class/net/eth1/device/sriov_numvfs\n",
:group => 'root',
:mode => '0755',

29
templates/switchdev/switchdev.epp
View File

@ -1,17 +1,30 @@
<%- | String $content_string = '', String $interface = '', String $count = '' | -%>
<%=$content_string%>set -eux
<%=$content_string%>set -ex
set -o pipefail
if [ "<%=$interface%>" == "$1" ]
then
exec 1> >(logger -s -t $(basename $0)) 2>&1
vendor_id="$(cat /sys/class/net/<%=$interface%>/device/vendor)"
if [[ $vendor_id == "0x15b3" && $(/sbin/lsmod | grep bonding) ]]
then
declare -A bonds
for bond in `cat /sys/class/net/bonding_masters`
do
bonds[$bond]=`cat "/sys/class/net/$bond/bonding/slaves"`
for bond_interface in ${bonds[$bond]}
do
echo "-$bond_interface" > "/sys/class/net/$bond/bonding/slaves"
done
done
fi
if [ "$(cat /sys/class/net/<%=$interface%>/device/sriov_numvfs)" == "0" ]
then
echo <%=$count%> > /sys/class/net/<%=$interface%>/device/sriov_numvfs
else
exit 0
fi
if [ "$(cat /sys/class/net/<%=$interface%>/device/vendor)" == "0x15b3" ]
if [ $vendor_id == "0x15b3" ]
then
vfs_pci_list=$(grep PCI_SLOT_NAME /sys/class/net/<%=$interface%>/device/virtfn*/uevent | cut -d'=' -f2)
for pci in $vfs_pci_list
@ -34,11 +47,21 @@ then
/usr/sbin/devlink dev eswitch set pci/"$interface_pci" inline-mode transport
fi
/usr/sbin/ethtool -K <%=$interface%> hw-tc-offload on
if [ "$(cat /sys/class/net/<%=$interface%>/device/vendor)" == "0x15b3" ]
if [ $vendor_id == "0x15b3" ]
then
for pci in $vfs_pci_list
do
echo "$pci" > /sys/bus/pci/drivers/mlx5_core/bind
done
fi
if [[ $vendor_id == "0x15b3" && $(/sbin/lsmod | grep bonding) ]]
then
for bond in "${!bonds[@]}"
do
for bond_interface in ${bonds[$bond]}
do
echo "+$bond_interface" > "/sys/class/net/$bond/bonding/slaves"
done
done
fi
fi