Implement ConnectX-4 configurations
Add changes to implement features for ConnectX-4 adapters Change-Id: Ie66162a2cb37ea3c1766f20bf218fb239eca8cf3
This commit is contained in:
parent
8f7a03adf5
commit
42c47329fc
|
@ -86,7 +86,9 @@ readonly USER_NUM_OF_VFS=`get_mlnx_param num_of_vfs`
|
|||
readonly ISER=`get_mlnx_param iser`
|
||||
readonly MAX_VFS=62
|
||||
readonly MIN_VFS=1
|
||||
readonly CX='ConnectX3'
|
||||
readonly CX=`get_mlnx_param cx_card`
|
||||
readonly NETWORK_TYPE=`get_mlnx_param network_type`
|
||||
readonly PHYSICAL_PORT=`get_mlnx_param physical_port`
|
||||
readonly VXLAN_OFFLOADING=`get_mlnx_param vxlan_offloading`
|
||||
readonly ROLES=`get_param roles`
|
||||
readonly ROLE=`get_param role`
|
||||
|
|
|
@ -64,7 +64,12 @@ function configure_qos () {
|
|||
|
||||
case $SCRIPT_MODE in
|
||||
'configure')
|
||||
configure_qos
|
||||
if [ "$CX" == "ConnectX-3" ]; then
|
||||
configure_qos
|
||||
fi
|
||||
if [ "$CX" == "ConnectX-4" ]; then
|
||||
logger_print info "QoS is not implemented for ConnectX-4."
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
logger_print error "Unsupported execution mode ${SCRIPT_MODE}."
|
||||
|
|
|
@ -87,7 +87,8 @@ function install_ofed_without_fw_update () {
|
|||
|
||||
logger_print info "Installing OFED drivers"
|
||||
OFED_INSTALL_SCRIPT_CMD="/usr/bin/perl ${OFED_INSTALL_SCRIPT}"
|
||||
${OFED_INSTALL_SCRIPT_CMD} --force --enable-sriov --without-fw-update
|
||||
${OFED_INSTALL_SCRIPT_CMD} --force --without-fw-update
|
||||
|
||||
rc=$?
|
||||
if [ $rc -ne 0 ]; then
|
||||
logger_print error "Failed execute ${OFED_INSTALL_SCRIPT_CMD} error code ${rc}"
|
||||
|
@ -97,36 +98,8 @@ function install_ofed_without_fw_update () {
|
|||
fi
|
||||
}
|
||||
|
||||
function update_fw_if_not_oem () {
|
||||
BUS_ID=`lspci | grep -m 1 Mellanox | cut -d' ' -f1`
|
||||
if [ -z $BUS_ID ]; then
|
||||
logger_print info "Didn't find bus, skipping firmware upgrade"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
mstflint -d ${BUS_ID} q | grep -i PSID | grep MT_
|
||||
if [ $? -ne 0 ]; then
|
||||
logger_print info "Not Mellanox Card, skipping firmware upgrade"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
OFED_INSTALL_SCRIPT="${OFED_DIR}/mlnxofedinstall"
|
||||
if [ ! -f $OFED_INSTALL_SCRIPT ]; then
|
||||
logger_print error "Failed to find $OFED_INSTALL_SCRIPT"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
logger_print info "Updating FW on Mellanox HCA with BUS ID = ${BUS_ID}"
|
||||
|
||||
OFED_INSTALL_SCRIPT_CMD="/usr/bin/perl ${OFED_INSTALL_SCRIPT}"
|
||||
${OFED_INSTALL_SCRIPT_CMD} --force --enable-sriov --fw-update-only
|
||||
if [ $? -ne 0 ]; then
|
||||
logger_print error "Failed execute ${OFED_INSTALL_SCRIPT_CMD} error code $?"
|
||||
fi
|
||||
}
|
||||
|
||||
function enable_eipoib (){
|
||||
if [ $DRIVER == 'eth_ipoib' ]; then
|
||||
if [ $NETWORK_TYPE == 'infiniband' ]; then
|
||||
sed -i s/^E_IPOIB_LOAD.*$/E_IPOIB_LOAD=yes/g /etc/infiniband/openib.conf
|
||||
|
||||
# Set Buffers size
|
||||
|
@ -161,9 +134,9 @@ if ! is_ofed_installed; then
|
|||
enable_eipoib
|
||||
fi
|
||||
|
||||
# OEM cards require a different dedicated OFED build, this build doesn't
|
||||
# support them.
|
||||
update_fw_if_not_oem
|
||||
|
||||
# Decrease loglevels for prevent flooding kernel messages to console
|
||||
sysctl_conf set 'kernel.printk' '4 4 1 7'
|
||||
service openibd stop
|
||||
service openibd start
|
||||
|
||||
exit 0
|
||||
|
|
|
@ -16,6 +16,7 @@
|
|||
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
import yaml
|
||||
import glob
|
||||
import logging
|
||||
|
@ -24,7 +25,16 @@ import traceback
|
|||
MLNX_SECTION = 'mellanox-plugin'
|
||||
SETTINGS_FILE = '/etc/astute.yaml'
|
||||
PLUGIN_OVERRIDE_FILE = '/etc/hiera/override/plugins.yaml'
|
||||
MLNX_DRIVERS_LIST = ('mlx4_en', 'eth_ipoib')
|
||||
MLNX_DRIVERS_LIST = { 'ConnectX-3': {'eth_driver' : 'mlx4_en', 'ib_driver' : 'eth_ipoib'},
|
||||
'ConnectX-4': {'eth_driver' : 'mlx5_core', 'ib_driver' : 'eth_ipoib'}}
|
||||
MLNX_DRIVERS = set([MLNX_DRIVERS_LIST[card][net]
|
||||
for card in MLNX_DRIVERS_LIST
|
||||
for net in MLNX_DRIVERS_LIST[card]])
|
||||
ETH_DRIVERS = set([MLNX_DRIVERS_LIST[card][net]
|
||||
for card in MLNX_DRIVERS_LIST
|
||||
for net in MLNX_DRIVERS_LIST[card]
|
||||
if net == 'eth_driver'])
|
||||
IB_DRIVERS = MLNX_DRIVERS - ETH_DRIVERS
|
||||
ISER_IFC_NAME = 'mlnx_iser0'
|
||||
LOG_FILE = '/var/log/mellanox-plugin.log'
|
||||
|
||||
|
@ -63,6 +73,56 @@ class MellanoxSettings(object):
|
|||
ifc = mlnx_interfaces_section[network]['interface']
|
||||
return ifc
|
||||
|
||||
@classmethod
|
||||
def get_card_type(cls, driver):
|
||||
for card in MLNX_DRIVERS_LIST.keys():
|
||||
if driver in MLNX_DRIVERS_LIST[card].values():
|
||||
network_driver_type = MLNX_DRIVERS_LIST[card].keys()[MLNX_DRIVERS_LIST[card].values()\
|
||||
.index(driver)]
|
||||
return card
|
||||
|
||||
@classmethod
|
||||
def add_cx_card(cls):
|
||||
mlnx_interfaces = cls.mlnx_interfaces_section
|
||||
drivers = list()
|
||||
interfaces = list()
|
||||
mlnx = cls.get_mlnx_section()
|
||||
for network_type, ifc_dict in mlnx_interfaces.iteritems():
|
||||
if 'driver' in ifc_dict and network_type in ['private','management','storage']:
|
||||
drivers.append(ifc_dict['driver'])
|
||||
interfaces.append(ifc_dict['interface'])
|
||||
|
||||
drivers_set = list(set(drivers))
|
||||
interfaces_set = list(set(interfaces))
|
||||
if (len(drivers_set) > 1):
|
||||
logging.error("Multiple ConnectX adapters was found in this environment.")
|
||||
raise MellanoxSettingsException(
|
||||
"Multiple ConnectX adapters was found in this environment."
|
||||
)
|
||||
else:
|
||||
current_driver = drivers_set[0]
|
||||
mellanox_interface = interfaces_set[0]
|
||||
if current_driver in ETH_DRIVERS:
|
||||
mlnx['network_type'] = 'ethernet'
|
||||
mlnx['cx_card'] = cls.get_card_type(current_driver)
|
||||
elif current_driver in IB_DRIVERS:
|
||||
mlnx['network_type'] = 'infiniband'
|
||||
ibdev = os.popen('ibdev2netdev').readlines()
|
||||
if not ibdev:
|
||||
mlnx['cx_card'] = 'none'
|
||||
logging.error('Failed executing ibdev2netdev')
|
||||
return 0
|
||||
interface_line = [l for l in ibdev if mellanox_interface in l]
|
||||
if interface_line and 'mlx5' in interface_line.pop():
|
||||
mlnx['cx_card'] = 'ConnectX-4'
|
||||
else:
|
||||
mlnx['cx_card'] = 'ConnectX-3'
|
||||
|
||||
network_info_msg = 'Detected Network Type is: %s ', mlnx['network_type']
|
||||
card_info_msg = 'Detected Card Type is: %s ', mlnx['cx_card']
|
||||
logging.info(network_info_msg)
|
||||
logging.info(card_info_msg)
|
||||
|
||||
@classmethod
|
||||
def add_driver(cls):
|
||||
interfaces = cls.get_interfaces_section()
|
||||
|
@ -87,14 +147,14 @@ class MellanoxSettings(object):
|
|||
mlnx = cls.get_mlnx_section()
|
||||
|
||||
private_ifc = cls.get_interface_by_network('private')
|
||||
if mlnx['driver'] == 'eth_ipoib':
|
||||
if mlnx['driver'] == MLNX_DRIVERS_LIST[mlnx['cx_card']]['ib_driver']:
|
||||
if 'bus_info' not in interfaces[private_ifc]['vendor_specific']:
|
||||
raise MellanoxSettingsException(
|
||||
"Couldn't find 'bus_info' for interface "
|
||||
"{0}".format(private_ifc)
|
||||
)
|
||||
mlnx['physical_port'] = interfaces[private_ifc]['vendor_specific']['bus_info']
|
||||
elif mlnx['driver'] == 'mlx4_en':
|
||||
elif mlnx['driver'] == MLNX_DRIVERS_LIST[mlnx['cx_card']]['eth_driver']:
|
||||
mlnx['physical_port'] = private_ifc
|
||||
|
||||
@classmethod
|
||||
|
@ -110,7 +170,7 @@ class MellanoxSettings(object):
|
|||
raise MellanoxSettingsException(
|
||||
"Failed reading vlan for br-storage"
|
||||
)
|
||||
if mlnx['driver'] == 'eth_ipoib':
|
||||
if mlnx['driver'] == MLNX_DRIVERS_LIST[mlnx['cx_card']]['ib_driver']:
|
||||
pkey = format((int(vlan) ^ 0x8000),'04x')
|
||||
mlnx['storage_pkey'] = pkey
|
||||
|
||||
|
@ -124,9 +184,9 @@ class MellanoxSettings(object):
|
|||
def add_iser_interface_name(cls):
|
||||
mlnx = cls.get_mlnx_section()
|
||||
storage_ifc = cls.get_interface_by_network('storage')
|
||||
if mlnx['driver'] == 'mlx4_en':
|
||||
if mlnx['driver'] == MLNX_DRIVERS_LIST[mlnx['cx_card']]['eth_driver']:
|
||||
mlnx['iser_ifc_name'] = ISER_IFC_NAME
|
||||
elif mlnx['driver'] == 'eth_ipoib':
|
||||
elif mlnx['driver'] == MLNX_DRIVERS_LIST[mlnx['cx_card']]['ib_driver']:
|
||||
interfaces = cls.get_interfaces_section()
|
||||
mlnx['iser_ifc_name'] = interfaces[storage_ifc]['vendor_specific']['bus_info']
|
||||
else:
|
||||
|
@ -150,7 +210,7 @@ class MellanoxSettings(object):
|
|||
# Handle iSER interface with and w/o vlan tagging
|
||||
storage_vlan = mlnx.get('storage_vlan')
|
||||
storage_parent = cls.get_interface_by_network('storage')
|
||||
if storage_vlan and mlnx['driver'] == 'mlx4_en': # Use VLAN dev
|
||||
if storage_vlan and mlnx['driver'] == MLNX_DRIVERS_LIST[mlnx['cx_card']]['eth_driver']: # Use VLAN dev
|
||||
vlan_name = "{0}.{1}".format(ISER_IFC_NAME, storage_vlan)
|
||||
# Set storage rule to iSER interface vlan interface
|
||||
cls.data['network_scheme']['roles']['storage'] = vlan_name
|
||||
|
@ -212,9 +272,10 @@ class MellanoxSettings(object):
|
|||
# the main change will be here because it reads phy_interfaces
|
||||
mlnx_interfaces = cls.mlnx_interfaces_section
|
||||
drivers = list()
|
||||
mlnx = cls.get_mlnx_section()
|
||||
for network_type, ifc_dict in mlnx_interfaces.iteritems():
|
||||
if 'driver' in ifc_dict and \
|
||||
ifc_dict['driver'] in MLNX_DRIVERS_LIST:
|
||||
ifc_dict['driver'] in MLNX_DRIVERS_LIST[mlnx['cx_card']].values():
|
||||
drivers.append(ifc_dict['driver'])
|
||||
return list(set(drivers))
|
||||
|
||||
|
@ -232,6 +293,8 @@ class MellanoxSettings(object):
|
|||
|
||||
@classmethod
|
||||
def update_role_settings(cls):
|
||||
# detect ConnectX card
|
||||
cls.add_cx_card()
|
||||
# realize the driver in use (eth/ib)
|
||||
cls.add_driver()
|
||||
# decide the physical function for SR-IOV
|
||||
|
|
|
@ -21,22 +21,28 @@ source ./common
|
|||
if ([[ $ROLES == *compute* ]] && [[ ! $ROLES == "compute" ]]) \
|
||||
&& [ $SRIOV == true ] ; then
|
||||
|
||||
# Update VFs
|
||||
./sriov.sh configure
|
||||
if [ $CX == 'ConnectX-3' ]; then
|
||||
|
||||
# Kill tgt daemons if exists
|
||||
tgt_locks=`find /var/run/ -name tgtd* | wc -l`
|
||||
if [ $tgt_locks -ne 0 ];then
|
||||
\rm -f /var/run/tgtd* && killall -9 tgtd
|
||||
service tgt stop
|
||||
# Update VFs
|
||||
./sriov.sh configure
|
||||
|
||||
# Kill tgt daemons if exists
|
||||
tgt_locks=`find /var/run/ -name tgtd* | wc -l`
|
||||
if [ $tgt_locks -ne 0 ];then
|
||||
\rm -f /var/run/tgtd* && killall -9 tgtd
|
||||
service tgt stop
|
||||
fi
|
||||
|
||||
service openibd restart && service openvswitch-switch restart
|
||||
|
||||
if [ $tgt_locks -ne 0 ];then
|
||||
service tgt start
|
||||
fi
|
||||
fi
|
||||
|
||||
service openibd restart && service openvswitch-switch restart
|
||||
|
||||
if [ $tgt_locks -ne 0 ];then
|
||||
service tgt start
|
||||
if [ $CX == 'ConnectX-4' ]; then
|
||||
service openibd restart && service openvswitch-switch restart
|
||||
#sleep 10
|
||||
fi
|
||||
|
||||
# Verify VFs
|
||||
./sriov.sh validate
|
||||
|
||||
|
|
|
@ -5,7 +5,7 @@ $firewall_driver = 'neutron.agent.firewall.NoopFirewallDriver'
|
|||
$private_net = $quantum_settings['default_private_net']
|
||||
$roles = hiera('roles')
|
||||
|
||||
if ( $mlnx_driver == 'mlx4_en' ){
|
||||
if ( $mlnx['network_type'] == 'ethernet' and $mlnx['iser'] ){
|
||||
$exclude_vf = '0'
|
||||
} else {
|
||||
$exclude_vf = ''
|
||||
|
@ -16,12 +16,13 @@ if ($mlnx['sriov']) {
|
|||
physnet => $quantum_settings['predefined_networks'][$private_net]['L2']['physnet'],
|
||||
physifc => $mlnx['physical_port'],
|
||||
mlnx_driver => $mlnx['driver'],
|
||||
network_type => $mlnx['network_type'],
|
||||
firewall_driver => $firewall_driver,
|
||||
exclude_vf => $exclude_vf,
|
||||
}
|
||||
}
|
||||
|
||||
# Configure QoS for ETH
|
||||
# Configure QoS for ConnectX3 ETH
|
||||
if ( $mlnx['driver'] == 'mlx4_en' and $mlnx['mlnx_qos'] ) {
|
||||
class { 'mellanox_openstack::configure_qos' :
|
||||
mlnx_sriov => $mlnx['sriov'],
|
||||
|
|
|
@ -4,7 +4,7 @@ $eswitch_apply_profile_patch = 'True'
|
|||
$mechanism_drivers = 'openvswitch'
|
||||
$roles = hiera('roles')
|
||||
|
||||
# Configure QoS for ETH
|
||||
# Configure QoS for connectX3 ETH
|
||||
if ( $mlnx['driver'] == 'mlx4_en' and $mlnx['mlnx_qos'] ) {
|
||||
class { 'mellanox_openstack::configure_qos' :
|
||||
mlnx_sriov => $mlnx['sriov'],
|
||||
|
@ -20,6 +20,7 @@ if ($mlnx['sriov']) {
|
|||
eswitch_apply_profile_patch => $eswitch_apply_profile_patch,
|
||||
mechanism_drivers => $mechanism_drivers,
|
||||
mlnx_driver => $mlnx['driver'],
|
||||
network_type => $mlnx['network_type'],
|
||||
mlnx_sriov => $mlnx['sriov'],
|
||||
pci_vendor_devices => $pci_vendor_devices,
|
||||
agent_required => $agent_required,
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
$mlnx = hiera('mellanox-plugin')
|
||||
|
||||
if ($mlnx['iser'] and $mlnx['driver'] == 'mlx4_en') {
|
||||
if ($mlnx['iser'] and $mlnx['network_type'] == 'ethernet') {
|
||||
class { 'mellanox_openstack::iser_rename' :
|
||||
storage_parent => $mlnx['storage_parent'],
|
||||
iser_interface_name => $mlnx['iser_ifc_name'],
|
||||
|
|
|
@ -2,6 +2,7 @@ class mellanox_openstack::compute_sriov (
|
|||
$physnet,
|
||||
$physifc,
|
||||
$mlnx_driver,
|
||||
$network_type,
|
||||
$firewall_driver,
|
||||
$exclude_vf,
|
||||
) {
|
||||
|
@ -35,7 +36,7 @@ class mellanox_openstack::compute_sriov (
|
|||
value => $firewall_driver,
|
||||
}
|
||||
|
||||
if ( $mlnx_driver == 'mlx4_en' ){
|
||||
if ( $network_type == 'ethernet' ){
|
||||
package { $sriov_agent_package:
|
||||
ensure => installed,
|
||||
}
|
||||
|
|
|
@ -3,6 +3,7 @@ class mellanox_openstack::controller_sriov (
|
|||
$eswitch_apply_profile_patch,
|
||||
$mechanism_drivers,
|
||||
$mlnx_driver,
|
||||
$network_type,
|
||||
$mlnx_sriov,
|
||||
$pci_vendor_devices,
|
||||
$agent_required,
|
||||
|
@ -30,7 +31,7 @@ class mellanox_openstack::controller_sriov (
|
|||
$sdn_extra_mechanism_driver=''
|
||||
}
|
||||
|
||||
if ( $mlnx_driver == 'mlx4_en' ){
|
||||
if ( $network_type == 'ethernet' ){
|
||||
$ml2_extra_mechanism_driver = "${sdn_extra_mechanism_driver}sriovnicswitch"
|
||||
neutron_plugin_ml2 {
|
||||
'ml2/mechanism_drivers': value => "${ml2_extra_mechanism_driver},${mechanism_drivers}";
|
||||
|
|
|
@ -93,7 +93,16 @@ if [ $PROBED_PORT_NAME != $ISER_NAME ]; then
|
|||
fi
|
||||
|
||||
# restart OFED modules for udev changes to take effect
|
||||
modprobe -r mlx4_en && modprobe mlx4_en
|
||||
modprobe -r mlx4_en
|
||||
modprobe -r mlx5_ib && modprobe -r mlx5_core
|
||||
udevadm control --reload-rules
|
||||
udevadm trigger
|
||||
modprobe mlx4_en
|
||||
modprobe mlx5_ib && modprobe mlx5_core
|
||||
echo "#!/bin/bash" > /etc/network/if-up.d/iser_ifc
|
||||
echo "ifconfig $ISER_NAME up " >> /etc/network/if-up.d/iser_ifc
|
||||
ifup --all
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
logger_print error "Mellanox drivers restart failed."
|
||||
exit 1
|
||||
|
|
|
@ -26,16 +26,16 @@ readonly GRUB_FILE_CENTOS="/boot/grub/grub.conf"
|
|||
readonly GRUB_FILE_UBUNTU="/boot/grub/grub.cfg"
|
||||
|
||||
function get_port_type() {
|
||||
if [ $DRIVER == 'mlx4_en' ]; then
|
||||
if [ $NETWORK_TYPE == 'ethernet' ]; then
|
||||
port_type=2
|
||||
elif [ $DRIVER == 'eth_ipoib' ]; then
|
||||
else
|
||||
port_type=1
|
||||
fi
|
||||
echo $port_type
|
||||
}
|
||||
|
||||
function get_num_probe_vfs () {
|
||||
if [ $DRIVER == 'mlx4_en' ]; then
|
||||
if [ `get_port_type` -eq "2" ]; then
|
||||
probe_vfs=`calculate_total_vfs`
|
||||
else
|
||||
probe_vfs=0
|
||||
|
@ -65,7 +65,7 @@ function calculate_total_vfs () {
|
|||
fi
|
||||
|
||||
# Set Ethernet RDMA storage network
|
||||
if [ $ISER == true ] && [ $DRIVER == 'mlx4_en' ] \
|
||||
if [ $ISER == true ] && [ `get_port_type` -eq "2" ] \
|
||||
&& [ $num_of_vfs -eq 0 ]; then
|
||||
num_of_vfs=1
|
||||
fi
|
||||
|
@ -149,37 +149,42 @@ function set_kernel_params () {
|
|||
|
||||
function burn_vfs_in_fw () {
|
||||
total_vfs=$1
|
||||
# required for mlxconfig to discover mlnx devices
|
||||
service openibd start &>/dev/null
|
||||
service mst start &>/dev/null
|
||||
devices=$(mst status -v | grep $CX| grep pciconf | awk '{print $2}')
|
||||
for dev in $devices; do
|
||||
logger_print debug "device=$dev"
|
||||
mlxconfig -d $dev q | grep SRIOV | awk '{print $2}' | grep $SRIOV_ENABLED_FLAG &>/dev/null
|
||||
sriov_enabled=$?
|
||||
current_num_of_vfs=`mlxconfig -d $dev q | grep NUM_OF_VFS | awk '{print $2}'`
|
||||
if [ $sriov_enabled -eq 0 ] 2>/dev/null; then
|
||||
logger_print debug "Detected SR-IOV is already enabled"
|
||||
else
|
||||
logger_print debug "Detected SR-IOV is disabled"
|
||||
fi
|
||||
if [[ ! "$total_vfs" == "$current_num_of_vfs" ]] 2>/dev/null; then
|
||||
logger_print debug "Current allowed number of VFs is ${current_num_of_vfs}, required number is ${total_vfs}"
|
||||
logger_print debug "Trying mlxconfig -y -d ${dev} s SRIOV_EN=1 NUM_OF_VFS=${total_vfs}"
|
||||
mlxconfig -y -d $dev s SRIOV_EN=1 NUM_OF_VFS=$total_vfs 2>&1 >/dev/null
|
||||
if [ $? -ne 0 ]; then
|
||||
logger_print error "Failed changing number of VFs in FW for HCA ${dev}"
|
||||
if [ $CX == 'ConnectX-3' ]; then
|
||||
# required for mlxconfig to discover mlnx devices
|
||||
service openibd start &>/dev/null
|
||||
service mst start &>/dev/null
|
||||
devices=$(mst status -v | grep $CX| grep pciconf | awk '{print $2}')
|
||||
for dev in $devices; do
|
||||
logger_print debug "device=$dev"
|
||||
mlxconfig -d $dev q | grep SRIOV | awk '{print $2}' | grep $SRIOV_ENABLED_FLAG &>/dev/null
|
||||
sriov_enabled=$?
|
||||
current_num_of_vfs=`mlxconfig -d $dev q | grep NUM_OF_VFS | awk '{print $2}'`
|
||||
if [ $sriov_enabled -eq 0 ] 2>/dev/null; then
|
||||
logger_print debug "Detected SR-IOV is already enabled"
|
||||
else
|
||||
logger_print debug "Detected SR-IOV is disabled"
|
||||
fi
|
||||
else
|
||||
logger_print debug "Current number of VFs is correctly set to ${current_num_of_vfs} in FW."
|
||||
fi
|
||||
done
|
||||
service mst stop &>/dev/null
|
||||
if [[ ! "$total_vfs" == "$current_num_of_vfs" ]] 2>/dev/null; then
|
||||
logger_print debug "Current allowed number of VFs is ${current_num_of_vfs}, required number is ${total_vfs}"
|
||||
logger_print debug "Trying mlxconfig -y -d ${dev} s SRIOV_EN=1 NUM_OF_VFS=${total_vfs}"
|
||||
mlxconfig -y -d $dev s SRIOV_EN=1 NUM_OF_VFS=$total_vfs 2>&1 >/dev/null
|
||||
if [ $? -ne 0 ]; then
|
||||
logger_print error "Failed changing number of VFs in FW for HCA ${dev}"
|
||||
fi
|
||||
else
|
||||
logger_print debug "Current number of VFs is correctly set to ${current_num_of_vfs} in FW."
|
||||
fi
|
||||
done
|
||||
service mst stop &>/dev/null
|
||||
fi
|
||||
if [ $CX == 'ConnectX-4' ]; then
|
||||
logger_print debug "Skipping burning ConnectX-4 as it is burnt in bootstrap stage."
|
||||
fi
|
||||
}
|
||||
|
||||
function is_sriov_required () {
|
||||
[ $SRIOV == true ] ||
|
||||
( [ $ISER == true ] && [ $DRIVER == 'mlx4_en' ] )
|
||||
( [ $ISER == true ] && [ `get_port_type` -eq "2" ] )
|
||||
return $?
|
||||
}
|
||||
|
||||
|
@ -195,9 +200,18 @@ function configure_sriov () {
|
|||
|
||||
probe_vfs=`get_num_probe_vfs`
|
||||
port_type=`get_port_type`
|
||||
set_modprobe_file $total_vfs &&
|
||||
set_kernel_params &&
|
||||
burn_vfs_in_fw $total_vfs
|
||||
if [ $CX == 'ConnectX-3' ]; then
|
||||
set_modprobe_file $total_vfs &&
|
||||
logger_print info "Detected: ConnectX-3 card"
|
||||
fi
|
||||
|
||||
if [ $CX == 'ConnectX-4' ]; then
|
||||
set_sriov $total_vfs &&
|
||||
logger_print info "Detected: ConnectX-4 card"
|
||||
fi
|
||||
|
||||
return $?
|
||||
else
|
||||
logger_print info "Skipping SR-IOV configuration"
|
||||
|
@ -236,8 +250,15 @@ function validate_sriov () {
|
|||
logger_print error "Failed , trying to fallback to ${FALLBACK_NUM_VFS}"
|
||||
probe_vfs=`get_num_probe_vfs`
|
||||
port_type=`get_port_type`
|
||||
set_modprobe_file $FALLBACK_NUM_VFS
|
||||
service openibd restart &> /dev/null
|
||||
|
||||
if [ $CX == 'ConnectX-3' ]; then
|
||||
set_modprobe_file $FALLBACK_NUM_VFS
|
||||
service openibd restart &> /dev/null
|
||||
fi
|
||||
if [ $CX == 'ConnectX-4' ]; then
|
||||
set_sriov $FALLBACK_NUM_VFS
|
||||
fi
|
||||
|
||||
current_num_vfs=`lspci | grep -i mellanox | grep -i virtual | wc -l`
|
||||
if [ $current_num_vfs -eq $FALLBACK_NUM_VFS ]; then
|
||||
logger_print info "Fallback to ${FALLBACK_NUM_VFS} succeeded"
|
||||
|
@ -248,11 +269,38 @@ function validate_sriov () {
|
|||
fi
|
||||
}
|
||||
|
||||
function set_sriov () {
|
||||
PORT_TYPE=`get_port_type`
|
||||
TOTAL_VFS=$1
|
||||
device_up=$PHYSICAL_PORT
|
||||
|
||||
if [ ${#device_up} -eq 0 ]; then
|
||||
logger_print error "Failed to find mlx5 up ports in ibdev2netdev."
|
||||
exit 1
|
||||
else
|
||||
res=`echo 0 > /sys/class/net/${device_up}/device/mlx5_num_vfs`
|
||||
res=`echo ${TOTAL_VFS} > /sys/class/net/${device_up}/device/mlx5_num_vfs`
|
||||
if [ ! $? -eq 0 ]; then
|
||||
logger_print error "Failed to write $TOTAL_VFS > /sys/class/net/${device_up}/device/mlx5_num_vfs"
|
||||
exit 1
|
||||
fi
|
||||
echo "#!/bin/bash" > /etc/network/if-up.d/sriov_vfs
|
||||
echo "echo ${TOTAL_VFS} > /sys/class/net/${device_up}/device/mlx5_num_vfs" >> /etc/network/if-up.d/sriov_vfs
|
||||
chmod +x /etc/network/if-up.d/sriov_vfs
|
||||
ifup --all
|
||||
if [ ! $? -eq 0 ]; then
|
||||
logger_print error "Failed to write $TOTAL_VFS > /sys/class/net/${device_up}/device/mlx5_num_vfs"
|
||||
exit 1
|
||||
else
|
||||
logger_print debug "Configured total vfs ${TOTAL_VFS} on ${device_up}"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
#################
|
||||
|
||||
case $SCRIPT_MODE in
|
||||
'configure')
|
||||
configure_sriov
|
||||
configure_sriov
|
||||
;;
|
||||
'validate')
|
||||
validate_sriov
|
||||
|
|
|
@ -20,16 +20,16 @@ source $SCRIPT_DIR/common
|
|||
readonly SCRIPT_MODE=$1
|
||||
|
||||
function get_port_type() {
|
||||
if [ $DRIVER == 'mlx4_en' ]; then
|
||||
if [ $NETWORK_TYPE == 'ethernet' ]; then
|
||||
port_type=2
|
||||
elif [ $DRIVER == 'eth_ipoib' ]; then
|
||||
else
|
||||
port_type=1
|
||||
fi
|
||||
echo $port_type
|
||||
}
|
||||
|
||||
function is_vxlan_offloading_required () {
|
||||
[ $VXLAN_OFFLOADING == true ] && [ $DRIVER == 'mlx4_en' ]
|
||||
[ $VXLAN_OFFLOADING == true ] && [ $NETWORK_TYPE == 'ethernet' ]
|
||||
return $?
|
||||
}
|
||||
|
||||
|
@ -69,7 +69,13 @@ function set_modprobe_file () {
|
|||
configure_vxlan_offloading
|
||||
case $SCRIPT_MODE in
|
||||
'configure')
|
||||
configure_vxlan_offloading
|
||||
if [ "$CX" == "ConnectX3" ]; then
|
||||
configure_vxlan_offloading
|
||||
fi
|
||||
if [ "$CX" == "ConnectX4" ]; then
|
||||
logger_print info "Skipping VXLAN configuration because VXLAN is configured with ConnectX4."
|
||||
fi
|
||||
|
||||
;;
|
||||
'validate')
|
||||
# to be added later.
|
||||
|
|
|
@ -63,17 +63,27 @@
|
|||
# Install OFED + FW upgrade
|
||||
- id: install_mlnx_ofed
|
||||
role: '*'
|
||||
required_for: [configure_vxlan_offload]
|
||||
required_for: [customize_mellanox_hiera_post]
|
||||
requires: [validate_kernel_devel]
|
||||
type: shell
|
||||
parameters:
|
||||
cmd: ./install_ofed.sh
|
||||
timeout: 5000
|
||||
# Add relevant settings for Mellanox manifests to mellanox plugin section in
|
||||
# Hiera, to make the data easily accessible and independent of astute.yaml
|
||||
- id: customize_mellanox_hiera_post
|
||||
role: '*'
|
||||
required_for: [configure_vxlan_offload]
|
||||
requires: [install_mlnx_ofed]
|
||||
type: shell
|
||||
parameters:
|
||||
cmd: ./mellanox_settings.py
|
||||
timeout: 100
|
||||
# add VXLAN offloading parameters
|
||||
- id: configure_vxlan_offload
|
||||
role: '*'
|
||||
required_for: [configure_sriov]
|
||||
requires: [install_mlnx_ofed]
|
||||
requires: [customize_mellanox_hiera_post]
|
||||
type: shell
|
||||
parameters:
|
||||
cmd: ./vxlan_offloading.sh configure
|
||||
|
@ -87,34 +97,26 @@
|
|||
type: shell
|
||||
parameters:
|
||||
cmd: ./sriov.sh configure
|
||||
timeout: 200
|
||||
timeout: 300
|
||||
# if VXLAN is not required. Rename iSER interface for Eth mode
|
||||
- id: rename_iser_probe_vf
|
||||
role: '*'
|
||||
required_for: [reboot_after_ofed]
|
||||
required_for: [validate_sriov]
|
||||
requires: [configure_sriov]
|
||||
type: puppet
|
||||
parameters:
|
||||
puppet_manifest: puppet/manifests/iser_rename.pp
|
||||
puppet_modules: puppet/modules:/etc/puppet/modules
|
||||
timeout: 200
|
||||
# Reboot due to OFED installation / IOMMU configuration
|
||||
- id: reboot_after_ofed
|
||||
role: '*'
|
||||
required_for: [validate_sriov]
|
||||
requires: [rename_iser_probe_vf]
|
||||
type: reboot
|
||||
parameters:
|
||||
timeout: 5000
|
||||
timeout: 300
|
||||
# Check number of VFs
|
||||
- id: validate_sriov
|
||||
role: '*'
|
||||
required_for: [configure_qos]
|
||||
requires: [reboot_after_ofed]
|
||||
requires: [rename_iser_probe_vf]
|
||||
type: shell
|
||||
parameters:
|
||||
cmd: ./sriov.sh validate
|
||||
timeout: 200
|
||||
timeout: 300
|
||||
# Enable QoS if required
|
||||
- id: configure_qos
|
||||
role: '*'
|
||||
|
@ -162,7 +164,7 @@
|
|||
type: shell
|
||||
parameters:
|
||||
cmd: ./post_set_mellanox_interfaces.sh
|
||||
timeout: 200
|
||||
timeout: 500
|
||||
# Execute post_deployment manifest for each role
|
||||
- id: configure_mellanox_controller
|
||||
role: ['controller', 'primary-controller']
|
||||
|
|
|
@ -32,7 +32,7 @@ attributes:
|
|||
If selected, Neutron "Quality of Service" (QoS) will be enabled for SR-IOV ports over Mellanox HCAs.
|
||||
This feature is supported in Ethernet mode over Neutron with VLAN after SR-IOV direct port creation.
|
||||
weight: 30
|
||||
type: "checkbox"
|
||||
type: hidden
|
||||
restrictions:
|
||||
- condition: "settings:common.libvirt_type.value != 'kvm' or not (settings:mellanox-plugin.sriov.value == true) or not (cluster:net_provider == 'neutron' and networking_parameters:segmentation_type == 'vlan')"
|
||||
message: "For Mellanox QoS support in Neutron, hypervisor type should be KVM and the environment networking should be based on Neutron with VLAN segmentation with Mellanox SR-IOV support."
|
||||
|
|
|
@ -2,16 +2,16 @@
|
|||
name: mellanox-plugin
|
||||
|
||||
# Human-readable name for your plugin
|
||||
title: Mellanox Openstack Features
|
||||
title: Mellanox ConnectX-4 Openstack Features
|
||||
|
||||
# Plugin version
|
||||
version: 3.0.0
|
||||
version: 3.1.0
|
||||
|
||||
# Description
|
||||
description: Enable features over Mellanox hardware
|
||||
description: Enable features over Mellanox ConnectX-4 Adapters
|
||||
|
||||
# Required fuel version
|
||||
fuel_version: ['7.0', '8.0']
|
||||
fuel_version: ['8.0']
|
||||
|
||||
# Specify license of your plugin
|
||||
licenses: ['Apache License Version 2.0']
|
||||
|
@ -27,11 +27,6 @@ groups: ['storage::cinder', 'network', 'hypervisor']
|
|||
|
||||
# The plugin is compatible with releases in the list
|
||||
releases:
|
||||
- os: ubuntu
|
||||
version: 2015.1.0-7.0
|
||||
mode: ['ha']
|
||||
deployment_scripts_path: deployment_scripts/
|
||||
repository_path: repositories/ubuntu
|
||||
- os: ubuntu
|
||||
version: liberty-8.0
|
||||
mode: ['ha']
|
||||
|
|
|
@ -72,7 +72,7 @@ old_debs="${PLUGIN_DIR}/repositories/ubuntu/*.deb"
|
|||
deb_files="cirros-testvm-mellanox_0.3.2-ubuntu3_amd64.deb
|
||||
cirros-testvm-mellanox-ib_0.3.2-9_amd64.deb
|
||||
eswitchd_1.0.0-18_amd64.deb
|
||||
mlnx-ofed-fuel_3.1-1.5.5_amd64.deb
|
||||
mlnx-ofed-fuel_3.2-2.0.0.0_amd64.deb
|
||||
mlnx-ofed-kernel-dkms_3.1-OFED.3.1.1.0.3.1.g9032737_all.deb
|
||||
mlnx-ofed-kernel-utils_3.1-OFED.3.1.1.0.3.1.g9032737_amd64.deb
|
||||
lldpd_0.9.1-0_amd64.deb
|
||||
|
|
Loading…
Reference in New Issue