Fuel plugin for Mellanox support
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

sriov.sh 12KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358
  1. #!/bin/bash -x
  2. # Copyright 2016 Mellanox Technologies, Ltd
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License");
  5. # you may not use this file except in compliance with the License.
  6. # You may obtain a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS,
  12. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  13. # implied.
  14. # See the License for the specific language governing permissions and
  15. # limitations under the License.
  16. readonly SCRIPT_DIR=$(dirname "$0")
  17. source $SCRIPT_DIR/common
  18. readonly SCRIPT_MODE=$1
  19. readonly FALLBACK_NUM_VFS=8
  20. readonly SRIOV_ENABLED_FLAG=1
  21. readonly VF_MAC_CACHING_TIMEOUT=1
  22. readonly VF_MAC_CACHING_TIMEOUT_DEF=300
  23. readonly NEW_KERNEL_PARAM="intel_iommu=on"
  24. readonly GRUB_FILE_CENTOS="/boot/grub/grub.conf"
  25. readonly GRUB_FILE_UBUNTU="/boot/grub/grub.cfg"
  26. function get_port_type() {
  27. if [ $NETWORK_TYPE == 'ethernet' ]; then
  28. port_type=2
  29. else
  30. port_type=1
  31. fi
  32. echo $port_type
  33. }
  34. function get_num_probe_vfs () {
  35. if [ `get_port_type` -eq "2" ]; then
  36. probe_vfs=`calculate_total_vfs`
  37. else
  38. probe_vfs=0
  39. fi
  40. echo $probe_vfs
  41. }
  42. function calculate_total_vfs () {
  43. max_card_vfs=$MAX_VFS
  44. if [ $CX == "ConnectX-5" ]; then
  45. max_card_vfs=$MAX_VFS_CX5
  46. fi
  47. # validate num of vfs is an integer, 0 <= num <= 96
  48. if [ "${USER_NUM_OF_VFS}" -ne "${USER_NUM_OF_VFS}" ] 2>/dev/null ||
  49. [ "${USER_NUM_OF_VFS}" -gt ${max_card_vfs} ] ||
  50. [ "${USER_NUM_OF_VFS}" -lt ${MIN_VFS} ]; then
  51. logger_print error "Illegal number of VFs ${USER_NUM_OF_VFS}, value
  52. should be an integer between ${MIN_VFS},${max_card_vfs}"
  53. return 1
  54. fi
  55. num_of_vfs=0
  56. # Set Compute VFs storage network
  57. if [ $SRIOV == true ]; then
  58. # If ROLES not set and not controller or compute
  59. if ([ -z $ROLES ] && [[ ! $ROLE == *controller* ]]) || \
  60. ([ $ROLE == compute ] || [[ $ROLES == *compute* ]]); then
  61. num_of_vfs=${USER_NUM_OF_VFS}
  62. fi
  63. fi
  64. # Set Ethernet RDMA storage network
  65. if [ $ISER == true ] && [ `get_port_type` -eq "2" ] \
  66. && [ $num_of_vfs -eq 0 ]; then
  67. num_of_vfs=1
  68. fi
  69. # Enforce even num of vfs
  70. if [ $((${num_of_vfs} % 2)) -eq 1 ]; then
  71. let num_of_vfs="${num_of_vfs} + 1"
  72. fi
  73. echo ${num_of_vfs}
  74. }
  75. # Reduce mac caching time since VF is used for iSER with non permanent MAC
  76. function reduce_mac_caching_timeout () {
  77. probes=`get_num_probe_vfs`
  78. if [ $probes -ge 1 ]; then
  79. timeout=$VF_MAC_CACHING_TIMEOUT
  80. else
  81. timeout=$VF_MAC_CACHING_TIMEOUT_DEF
  82. fi
  83. sysctl_conf set 'net.ipv4.route.gc_timeout' "$timeout"
  84. }
  85. function is_vxlan_offloading_required () {
  86. [ $VXLAN_OFFLOADING == true ]
  87. return $?
  88. }
  89. function set_modprobe_file () {
  90. PROBE_VFS=`get_num_probe_vfs`
  91. MLX4_CORE_FILE="/etc/modprobe.d/mlx4_core.conf"
  92. PORT_TYPE=`get_port_type`
  93. MLX4_CORE_STR="options mlx4_core
  94. enable_64b_cqe_eqe=0"
  95. if [[ $DEBUG == "true" ]];then
  96. MLX4_CORE_STR="${MLX4_CORE_STR} debug_level=1"
  97. fi
  98. TOTAL_VFS=$1
  99. MLX4_CORE_STR="${MLX4_CORE_STR} port_type_array=${PORT_TYPE},${PORT_TYPE}"
  100. if [[ $TOTAL_VFS -gt 0 ]]; then
  101. if [ $PORT_TYPE -eq 1 ]; then
  102. num_vfs="${TOTAL_VFS}"
  103. probe_vf="${TOTAL_VFS}"
  104. else
  105. num_vfs="${TOTAL_VFS},0,0"
  106. probe_vf="${TOTAL_VFS},0,0"
  107. fi
  108. MLX4_CORE_STR="${MLX4_CORE_STR} num_vfs=$num_vfs"
  109. if [[ $PROBE_VFS -gt 0 ]]; then
  110. MLX4_CORE_STR="${MLX4_CORE_STR} probe_vf=$probe_vf"
  111. fi
  112. fi
  113. MLX4_CORE_STR="${MLX4_CORE_STR} log_num_mgm_entry_size=-1"
  114. echo ${MLX4_CORE_STR} > ${MLX4_CORE_FILE}
  115. }
  116. function set_kernel_params () {
  117. if [ "$DISTRO" == "redhat" ]; then
  118. grub_file=${GRUB_FILE_CENTOS}
  119. kernel_line=`egrep 'kernel\s+/vmlinuz' ${grub_file} | grep -v '#'`
  120. elif [ "$DISTRO" == "ubuntu" ]; then
  121. grub_file=${GRUB_FILE_UBUNTU}
  122. kernel_line=$(echo "$(egrep 'linux\s+/vmlinuz' ${grub_file} | grep -v '#')" | head -1)
  123. fi
  124. if [[ $? -ne 0 ]]; then
  125. echo "Couldn't find kernel line in grub file" >&2 && return 1
  126. fi
  127. if ! grep -q ${NEW_KERNEL_PARAM} ${grub_file} ; then
  128. line_num=$(echo "$(grep -n "${kernel_line}" ${grub_file} |cut -f1 -d: )" | head -1)
  129. new_kernel_line="${kernel_line} ${NEW_KERNEL_PARAM}"
  130. # delete original line
  131. sed -i "${line_num}d" ${grub_file}
  132. # insert the corrected line on the same line number
  133. sed -i "${line_num}i\ ${new_kernel_line}" ${grub_file}
  134. fi
  135. reduce_mac_caching_timeout
  136. }
  137. function burn_vfs_in_fw () {
  138. total_vfs=$1
  139. if [ $CX == 'ConnectX-3' ]; then
  140. # required for mlxconfig to discover mlnx devices
  141. service openibd start &>/dev/null
  142. service mst start &>/dev/null
  143. devices=$(mst status -v | grep $(echo $CX | tr -d '-')| grep pciconf | awk '{print $2}')
  144. for dev in $devices; do
  145. logger_print debug "device=$dev"
  146. mlxconfig -d $dev q | grep SRIOV | awk '{print $2}' | grep $SRIOV_ENABLED_FLAG &>/dev/null
  147. sriov_enabled=$?
  148. current_num_of_vfs=`mlxconfig -d $dev q | grep NUM_OF_VFS | awk '{print $2}'`
  149. if [ $sriov_enabled -eq 0 ] 2>/dev/null; then
  150. logger_print debug "Detected SR-IOV is already enabled"
  151. else
  152. logger_print debug "Detected SR-IOV is disabled"
  153. fi
  154. if [[ ! "$total_vfs" == "$current_num_of_vfs" ]] 2>/dev/null; then
  155. logger_print debug "Current allowed number of VFs is ${current_num_of_vfs}, required number is ${total_vfs}"
  156. logger_print debug "Trying mlxconfig -y -d ${dev} s SRIOV_EN=1 NUM_OF_VFS=${total_vfs}"
  157. mlxconfig -y -d $dev s SRIOV_EN=1 NUM_OF_VFS=$total_vfs 2>&1 >/dev/null
  158. if [ $? -ne 0 ]; then
  159. logger_print error "Failed changing number of VFs in FW for HCA ${dev}"
  160. fi
  161. else
  162. logger_print debug "Current number of VFs is correctly set to ${current_num_of_vfs} in FW."
  163. fi
  164. done
  165. service mst stop &>/dev/null
  166. fi
  167. if [ $CX == 'ConnectX-4' ] || [ $CX == 'ConnectX-5' ]; then
  168. # required for mlxconfig to discover mlnx devices
  169. service openibd start &>/dev/null
  170. service mst start &>/dev/null
  171. devices=$(mst status -v | grep $(echo $CX | tr -d '-') | grep pciconf | awk '{print $2}')
  172. for dev in $devices; do
  173. current_fw_vfs=`mlxconfig -d $dev q | grep NUM_OF_VFS | awk '{print $2}'`
  174. if [ "$total_vfs" -gt "$current_fw_vfs" ]; then
  175. logger_print debug "device=$dev"
  176. logger_print debug "Trying mlxconfig -d ${dev} -y set NUM_OF_VFS=${total_vfs}"
  177. mlxconfig -d $dev -y set NUM_OF_VFS=$total_vfs
  178. fi
  179. done
  180. fi
  181. }
  182. function is_sriov_required () {
  183. [ $SRIOV == true ] ||
  184. ( [ $ISER == true ] && [ `get_port_type` -eq "2" ] )
  185. return $?
  186. }
  187. function configure_sriov () {
  188. if is_sriov_required; then
  189. # Calculate the total amount of virtual functions, based on user seclection
  190. total_vfs=`calculate_total_vfs`
  191. if [ -z ${total_vfs} ]; then
  192. exit 1
  193. fi
  194. logger_print info "Configuring ${total_vfs} virtual functions
  195. (only even number is currently supported)"
  196. probe_vfs=`get_num_probe_vfs`
  197. port_type=`get_port_type`
  198. set_kernel_params &&
  199. burn_vfs_in_fw $total_vfs
  200. if [ $CX == 'ConnectX-3' ]; then
  201. set_modprobe_file $total_vfs &&
  202. logger_print info "Detected: ConnectX-3 card"
  203. fi
  204. if [ $CX == 'ConnectX-4' ] || [ $CX == 'ConnectX-5' ]; then
  205. set_sriov $total_vfs &&
  206. logger_print info "Detected: $CX card"
  207. fi
  208. return $?
  209. else
  210. logger_print info "Skipping SR-IOV configuration"
  211. return 0
  212. fi
  213. }
  214. function validate_sriov () {
  215. if ! is_sriov_required; then
  216. logger_print info "Skipping SR-IOV validation, no virtual functions required"
  217. return 0
  218. fi
  219. logger_print info "Validating SR-IOV is enabled, and the required
  220. amount of virtual functions exist"
  221. # get number of VFs
  222. current_num_vfs=`lspci | grep -i mellanox | grep -i virtual | wc -l`
  223. total_vfs=`calculate_total_vfs`
  224. if [ -z ${total_vfs} ]; then
  225. exit 1
  226. fi
  227. # check if kernel was loaded with the new parameter
  228. grep ${NEW_KERNEL_PARAM} /proc/cmdline
  229. has_kernel_param_status=$?
  230. if [ $has_kernel_param_status -eq 0 ]; then
  231. if [ $current_num_vfs -eq $total_vfs ]; then
  232. logger_print info "Successfully verified SR-IOV is enabled with ${current_num_vfs} VFs"
  233. return 0
  234. fi
  235. else
  236. logger_print error "Kernel did not come up with the kernel parameter: ${NEW_KERNEL_PARAM},
  237. SR-IOV configuration failed"
  238. return 1
  239. fi
  240. # fallback only if kernel param exists and amount of vfs is not as expcted
  241. logger_print error "Failed , trying to fallback to ${FALLBACK_NUM_VFS}"
  242. probe_vfs=`get_num_probe_vfs`
  243. port_type=`get_port_type`
  244. if [ $CX == 'ConnectX-3' ]; then
  245. set_modprobe_file $FALLBACK_NUM_VFS
  246. service openibd restart &> /dev/null
  247. fi
  248. if [ $CX == 'ConnectX-4' ] || [ $CX == 'ConnectX-5' ]; then
  249. set_sriov $FALLBACK_NUM_VFS
  250. fi
  251. current_num_vfs=`lspci | grep -i mellanox | grep -i virtual | wc -l`
  252. if [ $current_num_vfs -eq $FALLBACK_NUM_VFS ]; then
  253. logger_print info "Fallback to ${FALLBACK_NUM_VFS} succeeded"
  254. return 0
  255. else
  256. logger_print error "Failed to configure SR-IOV"
  257. return 1
  258. fi
  259. }
  260. function set_sriov () {
  261. PORT_TYPE=`get_port_type`
  262. TOTAL_VFS=$1
  263. device_up=$PHYSICAL_PORT
  264. if [ ${#device_up} -eq 0 ]; then
  265. logger_print error "Failed to find mlx5 up ports in ibdev2netdev."
  266. exit 1
  267. else
  268. if [ "$(lspci | grep -i mellanox | grep -i virtual | wc -l)" -ne "$TOTAL_VFS" ]; then
  269. if [ ! $REBOOT_REQUIRED == true ] || [ $SCRIPT_MODE == "validate" ] ; then
  270. res=`echo 0 > /sys/class/net/${device_up}/device/mlx5_num_vfs`
  271. res=`echo ${TOTAL_VFS} > /sys/class/net/${device_up}/device/mlx5_num_vfs`
  272. if [ ! $? -eq 0 ]; then
  273. logger_print error "Failed to write $TOTAL_VFS > /sys/class/net/${device_up}/device/mlx5_num_vfs"
  274. exit 1
  275. else
  276. logger_print debug "Configured total vfs ${TOTAL_VFS} on ${device_up}"
  277. fi
  278. # Give MACs to created VFs
  279. python ./configure_mellanox_vfs.py ${TOTAL_VFS}
  280. fi
  281. # Make number of VFs persistent
  282. persistent_ifup_script=/etc/network/if-up.d/persistent_mlnx_params
  283. echo "#!/bin/bash" > $persistent_ifup_script
  284. chmod +x $persistent_ifup_script
  285. echo "# VFs Persistency" >> $persistent_ifup_script
  286. echo "if ! lspci | grep -i mellanox | grep -i virtual; then" >> $persistent_ifup_script
  287. echo "echo 0 > /sys/class/net/${device_up}/device/mlx5_num_vfs" >> $persistent_ifup_script
  288. echo "echo ${TOTAL_VFS} > /sys/class/net/${device_up}/device/mlx5_num_vfs" >> $persistent_ifup_script
  289. echo "fi" >> $persistent_ifup_script
  290. # Make the MAC for iser vf persistent
  291. echo "# MACs Persistency" >> $persistent_ifup_script
  292. echo 'if ip link show | grep "vf 0"| grep "MAC 00:00:00:00:00:00"; then' >> $persistent_ifup_script
  293. echo "python /etc/fuel/plugins/mellanox-plugin-*/configure_mellanox_vfs.py ${TOTAL_VFS}" >> $persistent_ifup_script
  294. echo "fi" >> $persistent_ifup_script
  295. echo "if [ -f /etc/init.d/tgt ]; then /etc/init.d/tgt force-reload; else exit 0; fi" >> $persistent_ifup_script
  296. if [ $REBOOT_REQUIRED == true ] && [ $SCRIPT_MODE == "configure" ] ; then
  297. logger_print debug "Configured total vfs ${TOTAL_VFS} on ${device_up} will apply \
  298. on next reboot as reboot is required"
  299. fi
  300. if [ $SCRIPT_MODE == "validate" ] ; then
  301. logger_print debug "Configured total vfs ${TOTAL_VFS} on ${device_up}"
  302. fi
  303. fi
  304. fi
  305. }
  306. #################
  307. case $SCRIPT_MODE in
  308. 'configure')
  309. configure_sriov
  310. ;;
  311. 'validate')
  312. validate_sriov
  313. ;;
  314. *)
  315. logger_print error "Unsupported execution mode ${SCRIPT_MODE}"
  316. exit 1
  317. ;;
  318. esac
  319. # Setting interfaces up
  320. for interface in `ifconfig -a | sed 's/[ \t].*//;/^\(lo\|\)$/d' | \
  321. sed 's/://'`;do ifconfig $interface up; done
  322. exit $?