From 517345f0db3ec46cfa9c2e41b83c0bd713b9c6a5 Mon Sep 17 00:00:00 2001 From: Aviram Bar-Haim Date: Sun, 6 Mar 2016 21:09:59 +0200 Subject: [PATCH] Post setting Mellanox interfaces 1. Moving IB iser child to persistent configurations. 2. Resetting VFs num in non controller nodes. Change-Id: I00cd3038aaeade99047983513b7e1c8f886f38e7 --- deployment_scripts/common | 5 +- ...onf.sh => post_set_mellanox_interfaces.sh} | 19 ++++-- .../manifests/configure_iser_child.pp | 4 +- deployment_scripts/sriov.sh | 22 ++++--- deployment_tasks.yaml | 60 +++++++++---------- 5 files changed, 63 insertions(+), 47 deletions(-) rename deployment_scripts/{move_ubuntu_ib_interfaces_conf.sh => post_set_mellanox_interfaces.sh} (82%) diff --git a/deployment_scripts/common b/deployment_scripts/common index 7a18721..5c2a41b 100755 --- a/deployment_scripts/common +++ b/deployment_scripts/common @@ -84,9 +84,10 @@ readonly DRIVER=`get_mlnx_param driver` readonly SRIOV=`get_mlnx_param sriov` readonly USER_NUM_OF_VFS=`get_mlnx_param num_of_vfs` readonly ISER=`get_mlnx_param iser` -readonly MAX_VFS=128 +readonly MAX_VFS=62 readonly MIN_VFS=1 readonly VXLAN_OFFLOADING=`get_mlnx_param vxlan_offloading` -readonly ROLE=`get_param roles` +readonly ROLES=`get_param roles` +readonly ROLE=`get_param role` readonly DEBUG=`get_param debug` readonly QOS=`get_mlnx_param mlnx_qos` diff --git a/deployment_scripts/move_ubuntu_ib_interfaces_conf.sh b/deployment_scripts/post_set_mellanox_interfaces.sh similarity index 82% rename from deployment_scripts/move_ubuntu_ib_interfaces_conf.sh rename to deployment_scripts/post_set_mellanox_interfaces.sh index 7cbbfc3..d1b8ea6 100755 --- a/deployment_scripts/move_ubuntu_ib_interfaces_conf.sh +++ b/deployment_scripts/post_set_mellanox_interfaces.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2015 Mellanox Technologies, Ltd +# Copyright 2016 Mellanox Technologies, Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -26,6 +26,15 @@ if [ $DISTRO == 'ubuntu' ] && [ $DRIVER == 'eth_ipoib' ]; then cat /etc/network/interfaces.d/ifcfg-ib* >> /etc/network/interfaces \rm -f /etc/network/interfaces.d/ifcfg-ib0* fi +fi + +# Set correct VF number in multi role computes +# (e.g. cinder & compute) +if ([[ $ROLES == *compute* ]] && [[ ! $ROLES == "compute" ]]) \ + && [ $SRIOV == true ] ; then + + # Update VFs + ./sriov.sh configure # Kill tgt daemons if exists tgt_locks=`find /var/run/ -name tgtd* | wc -l` @@ -36,11 +45,11 @@ if [ $DISTRO == 'ubuntu' ] && [ $DRIVER == 'eth_ipoib' ]; then service openibd restart && service openvswitch-switch restart - if [[ $ROLE == *compute* ]] && [ -f /etc/init.d/nova-compute ]; then - service nova-compute restart - fi - if [ $tgt_locks -ne 0 ];then service tgt start fi + + # Verify VFs + ./sriov.sh validate + fi diff --git a/deployment_scripts/puppet/modules/mellanox_openstack/manifests/configure_iser_child.pp b/deployment_scripts/puppet/modules/mellanox_openstack/manifests/configure_iser_child.pp index 7b804be..74d0089 100644 --- a/deployment_scripts/puppet/modules/mellanox_openstack/manifests/configure_iser_child.pp +++ b/deployment_scripts/puppet/modules/mellanox_openstack/manifests/configure_iser_child.pp @@ -7,8 +7,8 @@ class mellanox_openstack::configure_iser_child ( ipaddr => none, method => static } ~> - exec { 'refresh-iser-parent': - command => "/sbin/ifdown $iser_parent ; /sbin/ifup $iser_parent", + exec { 'flush-iser-parent': + command => "/sbin/ip addr flush dev $iser_parent", refreshonly => true, } -> l23_stored_config { $iser_child: diff --git a/deployment_scripts/sriov.sh b/deployment_scripts/sriov.sh index 8676bf3..919dc08 100755 --- a/deployment_scripts/sriov.sh +++ b/deployment_scripts/sriov.sh @@ -1,5 +1,5 @@ #!/bin/bash -x -# Copyright 2015 Mellanox Technologies, Ltd +# Copyright 2016 Mellanox Technologies, Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -54,19 +54,25 @@ function calculate_total_vfs () { fi num_of_vfs=0 - # SR-IOV is enabled, the given number of VFs is used - # iSER is also enabled, the iSER VF is among the given SR-IOV VFs - if [ $SRIOV == true ] && [[ $ROLE == *compute* ]]; then - num_of_vfs=${USER_NUM_OF_VFS} + # Set Compute VFs storage network + if [ $SRIOV == true ]; then - # SR-IOV is disabled with iSER enabled, then use only the storage VF - elif [ $ISER == true ] && [ $DRIVER == 'mlx4_en' ]; then + # If ROLES not set and not controller or compute + if ([ -z $ROLES ] && [[ ! $ROLE == *controller* ]]) || \ + ([ $ROLE == compute ] || [[ $ROLES == *compute* ]]); then + num_of_vfs=${USER_NUM_OF_VFS} + fi + fi + + # Set Ethernet RDMA storage network + if [ $ISER == true ] && [ $DRIVER == 'mlx4_en' ] \ + && [ $num_of_vfs -eq 0 ]; then num_of_vfs=1 fi # Enforce even num of vfs if [ $((${num_of_vfs} % 2)) -eq 1 ]; then - let num_of_vfs="${num_of_vfs} + 1" # number of vfs is odd and <= 64, then +1 is legal + let num_of_vfs="${num_of_vfs} + 1" fi echo ${num_of_vfs} } diff --git a/deployment_tasks.yaml b/deployment_tasks.yaml index fc24f72..0b784da 100644 --- a/deployment_tasks.yaml +++ b/deployment_tasks.yaml @@ -136,17 +136,38 @@ # Log a notice about post_deployment start - id: print_stage_post_deployment role: '*' - required_for: [post_deployment_end] + required_for: [configure_mlnx_iser_ib] requires: [post_deployment_start] type: shell parameters: cmd: ./log_stage.sh post_deployment timeout: 20 +# Set storage network configurations on the IB child +# if exists +- id: configure_mlnx_iser_ib + role: '*' + required_for: [post_set_mellanox_interfaces] + requires: [print_stage_post_deployment] + type: puppet + parameters: + puppet_manifest: puppet/manifests/configure_iser_child.pp + puppet_modules: puppet/modules:/etc/puppet/modules + timeout: 200 +# Move Ubuntu iSER conigurations to /etc/network/interfaces +# For MLNX_OFED drivers and reset VFs number +- id: post_set_mellanox_interfaces + role: '*' + required_for: [configure_mellanox_neo] + requires: [configure_mlnx_iser_ib] + type: shell + parameters: + cmd: ./post_set_mellanox_interfaces.sh + timeout: 120 # Execute post_deployment manifest for each role - id: configure_mellanox_controller role: ['controller', 'primary-controller'] - required_for: [configure_mlnx_iser_ib] - requires: [print_stage_post_deployment] + required_for: [configure_mellanox_neo] + requires: [post_set_mellanox_interfaces] type: puppet parameters: puppet_manifest: puppet/manifests/controller.pp @@ -154,8 +175,8 @@ timeout: 2000 - id: configure_mellanox_compute role: ['compute'] - required_for: [configure_mlnx_iser_ib] - requires: [print_stage_post_deployment] + required_for: [set_apparmor] + requires: [post_set_mellanox_interfaces] type: puppet parameters: puppet_manifest: puppet/manifests/compute.pp @@ -163,49 +184,28 @@ timeout: 500 - id: configure_mellanox_cinder role: ['cinder'] - required_for: [configure_mlnx_iser_ib] - requires: [print_stage_post_deployment] + required_for: [configure_mellanox_neo] + requires: [post_set_mellanox_interfaces] type: puppet parameters: puppet_manifest: puppet/manifests/cinder.pp puppet_modules: puppet/modules:/etc/puppet/modules timeout: 500 -# Set storage network configurations on the IB child -# if exists -- id: configure_mlnx_iser_ib - role: '*' - required_for: [set_apparmor] - requires: [post_deployment_start] - type: puppet - parameters: - puppet_manifest: puppet/manifests/configure_iser_child.pp - puppet_modules: puppet/modules:/etc/puppet/modules - timeout: 200 # Add required exceptions for SR-IOV - id: set_apparmor role: ['compute'] - required_for: [move_ib_interfaces_to_persistent] + required_for: [configure_mellanox_neo] requires: [configure_mlnx_iser_ib] type: shell parameters: cmd: ./set_apparmor.sh timeout: 200 -# Move Ubuntu iSER conigurations to /etc/network/interfaces -# For MLNX_OFED drivers -- id: move_ib_interfaces_to_persistent - role: '*' - required_for: [configure_mellanox_neo] - requires: [set_apparmor] - type: shell - parameters: - cmd: ./move_ubuntu_ib_interfaces_conf.sh - timeout: 120 # Configure Mellanox NEO SDN controller for auto VLAN # provisioning - id: configure_mellanox_neo role: '*' required_for: [cp_sriov_user_scripts] - requires: [move_ib_interfaces_to_persistent] + requires: [post_set_mellanox_interfaces] type: puppet parameters: puppet_manifest: puppet/manifests/configure_mlnx_neo.pp