Post setting Mellanox interfaces

1. Moving IB iser child to persistent configurations.
2. Resetting VFs num in non controller nodes.

Change-Id: I00cd3038aaeade99047983513b7e1c8f886f38e7

deployment_scripts/common

@@ -84,9 +84,10 @@ readonly DRIVER=`get_mlnx_param driver`
 readonly SRIOV=`get_mlnx_param sriov`
 readonly USER_NUM_OF_VFS=`get_mlnx_param num_of_vfs`
 readonly ISER=`get_mlnx_param iser`
-readonly MAX_VFS=128
+readonly MAX_VFS=62
 readonly MIN_VFS=1
 readonly VXLAN_OFFLOADING=`get_mlnx_param vxlan_offloading`
-readonly ROLE=`get_param roles`
+readonly ROLES=`get_param roles`
+readonly ROLE=`get_param role`
 readonly DEBUG=`get_param debug`
 readonly QOS=`get_mlnx_param mlnx_qos`

deployment_scripts/post_set_mellanox_interfaces.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2015 Mellanox Technologies, Ltd
+# Copyright 2016 Mellanox Technologies, Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -26,6 +26,15 @@ if [ $DISTRO == 'ubuntu' ] && [ $DRIVER == 'eth_ipoib' ]; then
     cat /etc/network/interfaces.d/ifcfg-ib* >> /etc/network/interfaces
     \rm -f /etc/network/interfaces.d/ifcfg-ib0*
   fi
+fi
+
+# Set correct VF number in multi role computes
+# (e.g. cinder & compute)
+if ([[ $ROLES == *compute* ]] && [[ ! $ROLES == "compute" ]]) \
+    && [ $SRIOV == true ] ; then
+
+  # Update VFs
+  ./sriov.sh configure
 
   # Kill tgt daemons if exists
   tgt_locks=`find /var/run/ -name tgtd* | wc -l`
@@ -36,11 +45,11 @@ if [ $DISTRO == 'ubuntu' ] && [ $DRIVER == 'eth_ipoib' ]; then
 
   service openibd restart && service openvswitch-switch restart
 
-  if [[ $ROLE == *compute* ]] && [ -f /etc/init.d/nova-compute ]; then
-    service nova-compute restart
-  fi
-
   if [ $tgt_locks -ne 0 ];then
     service tgt start
   fi
+
+  # Verify VFs
+  ./sriov.sh validate
+
 fi

deployment_scripts/puppet/modules/mellanox_openstack/manifests/configure_iser_child.pp

@@ -7,8 +7,8 @@ class mellanox_openstack::configure_iser_child (
     ipaddr   => none,
     method   => static
   } ~>
-  exec { 'refresh-iser-parent':
-    command     => "/sbin/ifdown $iser_parent ; /sbin/ifup $iser_parent",
+  exec { 'flush-iser-parent':
+    command     => "/sbin/ip addr flush dev $iser_parent",
     refreshonly => true,
   } ->
   l23_stored_config { $iser_child:

deployment_scripts/sriov.sh

@@ -1,5 +1,5 @@
 #!/bin/bash -x
-# Copyright 2015 Mellanox Technologies, Ltd
+# Copyright 2016 Mellanox Technologies, Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -54,19 +54,25 @@ function calculate_total_vfs () {
   fi
   num_of_vfs=0
 
-  # SR-IOV is enabled, the given number of VFs is used
-  # iSER is also enabled, the iSER VF is among the given SR-IOV VFs
-  if [ $SRIOV == true ] && [[ $ROLE == *compute* ]]; then
-    num_of_vfs=${USER_NUM_OF_VFS}
+  # Set Compute VFs storage network
+  if [ $SRIOV == true ]; then
 
-  # SR-IOV is disabled with iSER enabled, then use only the storage VF
-  elif [ $ISER == true ] && [ $DRIVER == 'mlx4_en' ]; then
+    # If ROLES not set and not controller or compute
+    if ([ -z $ROLES ] && [[ ! $ROLE == *controller* ]]) || \
+       ([ $ROLE == compute ] || [[ $ROLES == *compute* ]]); then
+      num_of_vfs=${USER_NUM_OF_VFS}
+    fi
+  fi
+
+  # Set Ethernet RDMA storage network
+  if [ $ISER == true ] && [ $DRIVER == 'mlx4_en' ] \
+     && [ $num_of_vfs -eq 0 ]; then
     num_of_vfs=1
   fi
 
   # Enforce even num of vfs
   if [ $((${num_of_vfs} % 2)) -eq 1 ]; then
-    let num_of_vfs="${num_of_vfs} + 1" # number of vfs is odd and <= 64, then +1 is legal
+    let num_of_vfs="${num_of_vfs} + 1"
   fi
   echo ${num_of_vfs}
 }

deployment_tasks.yaml

@@ -136,17 +136,38 @@
 # Log a notice about post_deployment start
 - id: print_stage_post_deployment
   role: '*'
-  required_for: [post_deployment_end]
+  required_for: [configure_mlnx_iser_ib]
   requires: [post_deployment_start]
   type: shell
   parameters:
     cmd: ./log_stage.sh post_deployment
     timeout: 20
+# Set storage network configurations on the IB child
+# if exists
+- id: configure_mlnx_iser_ib
+  role: '*'
+  required_for: [post_set_mellanox_interfaces]
+  requires: [print_stage_post_deployment]
+  type: puppet
+  parameters:
+    puppet_manifest: puppet/manifests/configure_iser_child.pp
+    puppet_modules: puppet/modules:/etc/puppet/modules
+    timeout: 200
+# Move Ubuntu iSER conigurations to /etc/network/interfaces
+# For MLNX_OFED drivers and reset VFs number
+- id: post_set_mellanox_interfaces
+  role: '*'
+  required_for: [configure_mellanox_neo]
+  requires: [configure_mlnx_iser_ib]
+  type: shell
+  parameters:
+    cmd: ./post_set_mellanox_interfaces.sh
+    timeout: 120
 # Execute post_deployment manifest for each role
 - id: configure_mellanox_controller
   role: ['controller', 'primary-controller']
-  required_for: [configure_mlnx_iser_ib]
-  requires: [print_stage_post_deployment]
+  required_for: [configure_mellanox_neo]
+  requires: [post_set_mellanox_interfaces]
   type: puppet
   parameters:
     puppet_manifest: puppet/manifests/controller.pp
@@ -154,8 +175,8 @@
     timeout: 2000
 - id: configure_mellanox_compute
   role: ['compute']
-  required_for: [configure_mlnx_iser_ib]
-  requires: [print_stage_post_deployment]
+  required_for: [set_apparmor]
+  requires: [post_set_mellanox_interfaces]
   type: puppet
   parameters:
     puppet_manifest: puppet/manifests/compute.pp
@@ -163,49 +184,28 @@
     timeout: 500
 - id: configure_mellanox_cinder
   role: ['cinder']
-  required_for: [configure_mlnx_iser_ib]
-  requires: [print_stage_post_deployment]
+  required_for: [configure_mellanox_neo]
+  requires: [post_set_mellanox_interfaces]
   type: puppet
   parameters:
     puppet_manifest: puppet/manifests/cinder.pp
     puppet_modules: puppet/modules:/etc/puppet/modules
     timeout: 500
-# Set storage network configurations on the IB child
-# if exists
-- id: configure_mlnx_iser_ib
-  role: '*'
-  required_for: [set_apparmor]
-  requires: [post_deployment_start]
-  type: puppet
-  parameters:
-    puppet_manifest: puppet/manifests/configure_iser_child.pp
-    puppet_modules: puppet/modules:/etc/puppet/modules
-    timeout: 200
 # Add required exceptions for SR-IOV
 - id: set_apparmor
   role: ['compute']
-  required_for: [move_ib_interfaces_to_persistent]
+  required_for: [configure_mellanox_neo]
   requires: [configure_mlnx_iser_ib]
   type: shell
   parameters:
     cmd: ./set_apparmor.sh
     timeout: 200
-# Move Ubuntu iSER conigurations to /etc/network/interfaces
-# For MLNX_OFED drivers
-- id: move_ib_interfaces_to_persistent
-  role: '*'
-  required_for: [configure_mellanox_neo]
-  requires: [set_apparmor]
-  type: shell
-  parameters:
-    cmd: ./move_ubuntu_ib_interfaces_conf.sh
-    timeout: 120
 # Configure Mellanox NEO SDN controller for auto VLAN
 # provisioning
 - id: configure_mellanox_neo
   role: '*'
   required_for: [cp_sriov_user_scripts]
-  requires: [move_ib_interfaces_to_persistent]
+  requires: [post_set_mellanox_interfaces]
   type: puppet
   parameters:
     puppet_manifest: puppet/manifests/configure_mlnx_neo.pp