Browse Source

Set VFs MAC addresses

Added mac address assignment for created mellanox vfs

Change-Id: Ia4045b0799eacbfc01ceebf8d0d23c8db36f8197
tags/3.2.0-1
Rawan Herzallah 3 years ago
parent
commit
8b30f4cb75

+ 204
- 0
deployment_scripts/configure_mellanox_vfs.py View File

@@ -0,0 +1,204 @@
1
+#!/usr/bin/env python
2
+# Copyright 2016 Mellanox Technologies, Ltd
3
+#
4
+# Licensed under the Apache License, Version 2.0 (the "License");
5
+# you may not use this file except in compliance with the License.
6
+# You may obtain a copy of the License at
7
+#
8
+#    http://www.apache.org/licenses/LICENSE-2.0
9
+#
10
+# Unless required by applicable law or agreed to in writing, software
11
+# distributed under the License is distributed on an "AS IS" BASIS,
12
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
13
+# implied.
14
+# See the License for the specific language governing permissions and
15
+# limitations under the License.
16
+
17
+import os
18
+import sys
19
+import subprocess
20
+import logging
21
+import traceback
22
+import glob
23
+
24
+LOG_FILE = '/var/log/mellanox-plugin.log'
25
+
26
+class MellanoxVfsSettingsException(Exception):
27
+    pass
28
+
29
+class MellanoxVfsSettings(object):
30
+
31
+    mellanox_vfs = list();
32
+
33
+    @classmethod
34
+    def get_pci_list(cls):
35
+        cmd_vfs = "lspci -D | grep -i mellanox | grep -i virtual | awk '{print $1}'"
36
+        p_lspci = subprocess.Popen(cmd_vfs, shell=True, stdout=subprocess.PIPE,
37
+                                   stderr=subprocess.STDOUT)
38
+        retval = p_lspci.wait()
39
+        if retval != 0:
40
+            print 'Failed to execute command lspci.'
41
+            logging.error("Failed to find mellanox VFs.")
42
+            sys.exit(1)
43
+        pci_res = list()
44
+        for line in p_lspci.stdout.readlines():
45
+            pci_res.append(line.rstrip())
46
+        return pci_res
47
+
48
+    @classmethod
49
+    def build_vfs_dict(cls):
50
+        pci_res = cls.get_pci_list()
51
+        cmd_vfs_data = "ibdev2netdev -v"
52
+        p_vfs_data = subprocess.Popen(cmd_vfs_data, shell=True, stdout=subprocess.PIPE,
53
+                                      stderr=subprocess.STDOUT)
54
+        retval = p_vfs_data.wait()
55
+        if retval != 0:
56
+            print 'Failed to execute command ibdev2netdev -v.'
57
+            logging.error("Failed to execute ibdev2netdev -v")
58
+            sys.exit(1)
59
+
60
+        ibdev = list()
61
+        for vf_data in p_vfs_data.stdout.readlines():
62
+            ibdev.append(vf_data.rstrip())
63
+
64
+        cmd_vfs_macs = "ip link show | grep vf | awk '{print $4}' | tr -d ','"
65
+        p_vfs_macs = subprocess.Popen(cmd_vfs_macs, shell=True, stdout=subprocess.PIPE,
66
+                                      stderr=subprocess.STDOUT)
67
+        retval = p_vfs_macs.wait()
68
+        if retval != 0:
69
+            print 'Failed to execute command ip link show | grep vf '
70
+            logging.error("Failed to find VFs in ip link show command")
71
+            sys.exit(1)
72
+
73
+        mac_list = list()
74
+        for mac in p_vfs_macs.stdout.readlines():
75
+            mac_list.append(mac.rstrip())
76
+
77
+        count = 0
78
+        for pci_address in pci_res:
79
+            vf_dict = dict();
80
+            vf_info = next((vf_info for vf_info in ibdev if pci_address in vf_info), None)
81
+            vf_info = vf_info.split();
82
+            vf_dict['vf_num'] = count;
83
+            vf_dict['pci_address'] = vf_info[0]
84
+            vf_dict['port_module'] = vf_info[1]
85
+            vf_dict['port_num'] = vf_info[12]
86
+            vf_dict['mac'] = mac_list[count]
87
+            cls.mellanox_vfs.append(vf_dict)
88
+            count += 1;
89
+
90
+    @classmethod
91
+    def unbind(cls):
92
+        for vf in cls.mellanox_vfs:
93
+            cmd_vfs_pci = "echo {0} ".format(vf['pci_address']) + \
94
+                           ">> /sys/bus/pci/drivers/mlx5_core/unbind"
95
+            p_unbind_pci = subprocess.Popen(cmd_vfs_pci, shell=True, stdout=subprocess.PIPE,
96
+                                            stderr=subprocess.STDOUT)
97
+            retval = p_unbind_pci.wait()
98
+            if retval != 0:
99
+                logging.error("Failed to unbind pci address {0}".format(vf['pci_address']))
100
+                sys.exit(1)
101
+        logging.info("Managed to unbind VFs.")
102
+
103
+    @classmethod
104
+    def bind(cls):
105
+        for vf in cls.mellanox_vfs:
106
+            cmd_vfs_pci = "echo {0} ".format(vf['pci_address']) + \
107
+                           ">> /sys/bus/pci/drivers/mlx5_core/bind"
108
+            p_bind_pci = subprocess.Popen(cmd_vfs_pci, shell=True, stdout=subprocess.PIPE,
109
+                                          stderr=subprocess.STDOUT)
110
+            retval = p_bind_pci.wait()
111
+            if retval != 0:
112
+                print 'Bind: Error is:', p_bind_pci.stdout.readlines()
113
+                logging.error("Failed to bind pci address {0}".format(vf['pci_address']) )
114
+                sys.exit(1)
115
+            else:
116
+                print 'Managed to bind pci address ', vf['pci_address']
117
+                logging.debug("Managed to bind pci address {0}".format(vf['pci_address']) )
118
+        logging.info("Managed to bind VFs.")
119
+
120
+    @classmethod
121
+    def assign_mac_per_vf(cls):
122
+        for vf in cls.mellanox_vfs:
123
+            if "00:00:00:00:00:00" in vf['mac']:
124
+                cmd_generate_mac = "ibstat {0} {1} |".format(vf['port_module'], vf['port_num']) + \
125
+                                   " grep GUID | cut -d' ' -f3 | cut -d'x' -f2"
126
+                p_cmd_generate_mac = subprocess.Popen(cmd_generate_mac, shell=True,
127
+                                                      stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
128
+                retval = p_cmd_generate_mac.wait()
129
+                if retval != 0:
130
+                     print 'Failed to get ibstat port guid'
131
+                     logging.error("Failed to find PORT GUID to set it as a MAC address for the VF")
132
+                     sys.exit(1)
133
+                port_guid = p_cmd_generate_mac.stdout.readlines()[0].rstrip();
134
+                port_guid_to_mac = port_guid[0:12]
135
+                port_guid_to_mac = ':'.join(port_guid_to_mac[i:i+2] for i in range \
136
+                                   (0, len(port_guid_to_mac), 2))
137
+                vf['mac'] = port_guid_to_mac
138
+
139
+    @classmethod
140
+    def set_mac_per_vf(cls):
141
+        cmd_physical_port = "hiera mellanox-plugin | grep physical_port | cut -d'>' -f2 "
142
+        p = subprocess.Popen(cmd_physical_port ,shell=True,stdout=subprocess.PIPE,
143
+                             stderr=subprocess.STDOUT)
144
+        physical_port = p.stdout.readlines()[0].rstrip()
145
+        physical_port = physical_port.strip(',"')
146
+        for vf in cls.mellanox_vfs:
147
+            cmd_set_mac_per_vf = "ip link set " + \
148
+                                 "{0} vf {1} mac {2}".format(physical_port,vf['vf_num'], vf['mac'])
149
+            p_cmd_set_mac_per_vf =  subprocess.Popen(cmd_set_mac_per_vf,shell=True,
150
+                                                     stdout=subprocess.PIPE,
151
+                                                     stderr=subprocess.STDOUT)
152
+            retval = p_cmd_set_mac_per_vf.wait()
153
+            if retval != 0:
154
+                 print 'Failed to set vf mac', cmd_set_mac_per_vf
155
+                 logging.error("Failed to set MAC address to VF {0}".format(vf['vf_num']))
156
+                 sys.exit(1)
157
+
158
+    @classmethod
159
+    def wait_for_mlx_modules_and_vfs_loaded(cls, total_vfs):
160
+        retval = 1
161
+        while (retval != 0):
162
+            cmd_lsmod = "lsmod | grep mlx_compat"
163
+            p_lsmod = subprocess.Popen(cmd_lsmod, shell=True,stdout=subprocess.PIPE,
164
+                                       stderr=subprocess.STDOUT)
165
+            retval = p_lsmod.wait()
166
+        logging.info("mlx_compat module is loaded!")
167
+
168
+        number_of_vfs = 0
169
+        while ( number_of_vfs != total_vfs):
170
+            cmd_number_vfs = "lspci | grep -i mellanox | grep -i virtual | wc -l"
171
+            p_number_vfs = subprocess.Popen(cmd_number_vfs, shell=True,stdout=subprocess.PIPE,
172
+                                            stderr=subprocess.STDOUT)
173
+            retval = p_number_vfs.wait()
174
+            if retval != 0:
175
+                logging.error("Failed to run lspci")
176
+                sys.exit(1)
177
+            number_of_vfs = int(p_number_vfs.stdout.readlines()[0].rstrip());
178
+        logging.info("all VFs are loaded.")
179
+
180
+def main(total_vfs):
181
+    logging.basicConfig(format='%(asctime)s %(message)s',
182
+                        level=logging.DEBUG, filename=LOG_FILE)
183
+    try:
184
+        vfs_configurations = MellanoxVfsSettings()
185
+        vfs_configurations.wait_for_mlx_modules_and_vfs_loaded(int(total_vfs))
186
+        vfs_configurations.build_vfs_dict()
187
+        vfs_configurations.assign_mac_per_vf()
188
+        vfs_configurations.unbind()
189
+        vfs_configurations.set_mac_per_vf()
190
+        vfs_configurations.bind()
191
+
192
+    except MellanoxVfsSettingsException, exc:
193
+        error_msg = "Failed configuring Mellanox vfs: {0}\n".format(exc)
194
+        sys.stderr.write(error_msg)
195
+        logging.error(exc)
196
+        sys.exit(1)
197
+    success_msg = "Done configuring Mellanox vfs\n"
198
+    sys.stdout.write(success_msg)
199
+    logging.info(success_msg)
200
+    sys.exit(0)
201
+
202
+
203
+if __name__=="__main__":
204
+    main(sys.argv[1])

+ 8
- 0
deployment_scripts/install_ofed.sh View File

@@ -119,6 +119,10 @@ function enable_eipoib (){
119 119
   fi
120 120
 }
121 121
 
122
+function install_gawk (){
123
+  apt-get -y install gawk
124
+}
125
+
122 126
 if ! is_ofed_installed; then
123 127
   # Install mlnx-ofed-fuel rpm/deb package which extracts OFED installation dir
124 128
   install_mlnx_ofed_src
@@ -132,6 +136,10 @@ if ! is_ofed_installed; then
132 136
 
133 137
   # Enable Ethernet IP Over Infiniband in case of eth_ipoib driver
134 138
   enable_eipoib
139
+
140
+  # Install gawk package for running ibdev2netdev -v
141
+  install_gawk
142
+
135 143
 fi
136 144
 
137 145
 # Decrease loglevels for prevent flooding kernel messages to console

+ 0
- 3
deployment_scripts/puppet/modules/mellanox_openstack/templates/iser_rename.erb View File

@@ -99,9 +99,6 @@ if [ $PROBED_PORT_NAME != $ISER_NAME ]; then
99 99
   udevadm trigger
100 100
   modprobe mlx4_en
101 101
   modprobe mlx5_ib && modprobe mlx5_core
102
-  echo "#!/bin/bash" > /etc/network/if-up.d/iser_ifc
103
-  echo "ifconfig $ISER_NAME up " >> /etc/network/if-up.d/iser_ifc
104
-  ifup --all
105 102
 
106 103
   if [ $? -ne 0 ]; then
107 104
     logger_print error "Mellanox drivers restart failed."

+ 28
- 16
deployment_scripts/sriov.sh View File

@@ -278,21 +278,33 @@ function set_sriov () {
278 278
     logger_print error "Failed to find mlx5 up ports in ibdev2netdev."
279 279
     exit 1
280 280
   else
281
-    res=`echo 0 > /sys/class/net/${device_up}/device/mlx5_num_vfs`
282
-    res=`echo ${TOTAL_VFS} > /sys/class/net/${device_up}/device/mlx5_num_vfs`
283
-    if [ ! $? -eq 0 ]; then
284
-      logger_print error "Failed to write $TOTAL_VFS > /sys/class/net/${device_up}/device/mlx5_num_vfs"
285
-      exit 1
286
-    fi
287
-    echo "#!/bin/bash" > /etc/network/if-up.d/sriov_vfs
288
-    echo "echo ${TOTAL_VFS} > /sys/class/net/${device_up}/device/mlx5_num_vfs" >> /etc/network/if-up.d/sriov_vfs
289
-    chmod +x /etc/network/if-up.d/sriov_vfs
290
-    ifup --all
291
-    if [ ! $? -eq 0 ]; then
292
-      logger_print error "Failed to write $TOTAL_VFS > /sys/class/net/${device_up}/device/mlx5_num_vfs"
293
-      exit 1
294
-    else
295
-      logger_print debug "Configured total vfs ${TOTAL_VFS} on ${device_up}"
281
+    if [ "$(lspci | grep -i mellanox | grep -i virtual | wc -l)" -ne "$TOTAL_VFS" ]; then
282
+      res=`echo 0 > /sys/class/net/${device_up}/device/mlx5_num_vfs`
283
+      res=`echo ${TOTAL_VFS} > /sys/class/net/${device_up}/device/mlx5_num_vfs`
284
+      if [ ! $? -eq 0 ]; then
285
+        logger_print error "Failed to write $TOTAL_VFS > /sys/class/net/${device_up}/device/mlx5_num_vfs"
286
+        exit 1
287
+      fi
288
+
289
+      # Give MACs to created VFs
290
+      python ./configure_mellanox_vfs.py ${TOTAL_VFS}
291
+
292
+      # Make number of VFs and their MACs persistent
293
+      persistent_ifup_script=/etc/network/if-up.d/persistent_mlnx_params
294
+      echo "#!/bin/bash" > $persistent_ifup_script
295
+      chmod +x $persistent_ifup_script
296
+      echo "if ! lspci | grep -i mellanox | grep -i virtual; then" >> $persistent_ifup_script
297
+      echo "echo ${TOTAL_VFS} > /sys/class/net/${device_up}/device/mlx5_num_vfs" >> $persistent_ifup_script
298
+      echo "python /etc/fuel/plugins/mellanox-plugin-*/configure_mellanox_vfs.py ${TOTAL_VFS}" >> $persistent_ifup_script
299
+      echo "fi" >> $persistent_ifup_script
300
+      echo "if [ -f /etc/init.d/tgt ]; then /etc/init.d/tgt force-reload; else exit 0; fi" >> $persistent_ifup_script
301
+
302
+      if [ ! $? -eq 0 ]; then
303
+        logger_print error "Failed to write $TOTAL_VFS > /sys/class/net/${device_up}/device/mlx5_num_vfs"
304
+        exit 1
305
+      else
306
+        logger_print debug "Configured total vfs ${TOTAL_VFS} on ${device_up}"
307
+      fi
296 308
     fi
297 309
   fi
298 310
 }
@@ -300,7 +312,7 @@ function set_sriov () {
300 312
 
301 313
 case $SCRIPT_MODE in
302 314
   'configure')
303
-    configure_sriov 
315
+    configure_sriov
304 316
     ;;
305 317
   'validate')
306 318
     validate_sriov

+ 1
- 1
metadata.yaml View File

@@ -5,7 +5,7 @@ name: mellanox-plugin
5 5
 title: Mellanox ConnectX-4 Openstack Features
6 6
 
7 7
 # Plugin version
8
-version: 3.1.5
8
+version: 3.1.8
9 9
 
10 10
 # Description
11 11
 description: Enable features over Mellanox ConnectX-4 Adapters

+ 1
- 1
pre_build_hook View File

@@ -62,7 +62,7 @@ old_debs="${PLUGIN_DIR}/repositories/ubuntu/*.deb"
62 62
 deb_files="cirros-testvm-mellanox_0.3.2-ubuntu3_amd64.deb
63 63
            cirros-testvm-mellanox-ib_0.3.2-9_amd64.deb
64 64
            eswitchd_1.0.0-18_amd64.deb
65
-           mlnx-ofed-fuel_3.2-2.0.0.0_amd64.deb
65
+           mlnx-ofed-fuel_3.3-0.1.0.0_amd64.deb
66 66
            lldpd_0.9.1-0_amd64.deb
67 67
            python-networking-mlnx_7.0.0-1_all.deb"
68 68
 get_packages "deb" "$old_debs" "$deb_files"

Loading…
Cancel
Save