Browse Source

Restructuring the method of vf-representor renaming

The old way of renmaing depended on adding udev rule
before creating vf-representors and so we faced race condition on
uplink representor rename.
So now we are creating vf-representors and after that we adding the
udev rules and trigerring them at the end
Also because the old name of UDEV_RULE_FILE(70-persistent-net.rules) is
used by another module, so we renamed the udev_rule_file to
(80-persistent-os-net-config.rules) and with that way it will override
any previouse rules in (70-persistent-net.rules) and will not any cause
any confustion
Addition to that, we also added a udev rule to make vf-representors
unmanaged by NetworkManager in order to be renamed successfully

Change-Id: I61c59e162bc5d15c063f52612e198d93f4433db1
Closes-Bug: #1830716
(cherry picked from commit 5d44b582b5)
tags/8.5.1
waleed mousa 3 months ago
parent
commit
bab5da768b
1 changed files with 97 additions and 29 deletions
  1. 97
    29
      os_net_config/sriov_config.py

+ 97
- 29
os_net_config/sriov_config.py View File

@@ -28,36 +28,26 @@ import pyudev
28 28
 import re
29 29
 from six.moves import queue as Queue
30 30
 import sys
31
+import time
31 32
 import yaml
32 33
 
33 34
 from oslo_concurrency import processutils
34 35
 
35 36
 logger = logging.getLogger(__name__)
36 37
 _SYS_CLASS_NET = '/sys/class/net'
37
-_UDEV_RULE_FILE = '/etc/udev/rules.d/70-persistent-net.rules'
38
-
38
+_UDEV_RULE_FILE = '/etc/udev/rules.d/80-persistent-os-net-config.rules'
39
+MAX_RETRIES = 10
40
+PF_FUNC_RE = re.compile(r"\.(\d+)$", 0)
39 41
 # In order to keep VF representor name consistent specially after the upgrade
40 42
 # proccess, we should have a udev rule to handle that.
41 43
 # The udev rule will rename the VF representor as "<sriov_pf_name>_<vf_num>"
42 44
 _REP_LINK_NAME_FILE = "/etc/udev/rep-link-name.sh"
43 45
 _REP_LINK_NAME_DATA = '''#!/bin/bash
44
-SWID="$1"
45
-PORT="$2"
46
-parent_phys_port_name=${PORT%vf*}
47
-parent_phys_port_name=${parent_phys_port_name//f}
48
-for i in `ls -1 /sys/class/net/*/phys_port_name`
49
-do
50
-    nic=`echo $i | cut -d/ -f 5`
51
-    sw_id=`cat /sys/class/net/$nic/phys_switch_id 2>/dev/null`
52
-    phys_port_name=`cat /sys/class/net/$nic/phys_port_name 2>/dev/null`
53
-    if [ "$parent_phys_port_name" = "$phys_port_name" ] &&
54
-       [ "$sw_id" = "$SWID" ]
55
-    then
56
-        echo "NAME=${nic}_${PORT##pf*vf}"
57
-        break
58
-        exit
59
-    fi
60
-done'''
46
+# This file is autogenerated by os-net-config
47
+set -x
48
+PORT="$1"
49
+echo "NUMBER=${PORT##pf*vf}"
50
+'''
61 51
 
62 52
 # Create a queue for passing the udev network events
63 53
 vf_queue = Queue.Queue()
@@ -123,6 +113,7 @@ def configure_sriov_pf():
123 113
     sriov_map = _get_sriov_map()
124 114
     MLNX_UNBIND_FILE_PATH = "/sys/bus/pci/drivers/mlx5_core/unbind"
125 115
     MLNX_VENDOR_ID = "0x15b3"
116
+    trigger_udev_rule = False
126 117
     for item in sriov_map:
127 118
         if item['device_type'] == 'pf':
128 119
             _pf_interface_up(item)
@@ -147,12 +138,27 @@ def configure_sriov_pf():
147 138
                     if os.path.exists(vf_pci_path):
148 139
                         with open(MLNX_UNBIND_FILE_PATH, 'w') as f:
149 140
                             f.write("%s" % vf_pci)
141
+
142
+                # Adding a udev rule to make vf-representors unmanaged by
143
+                # NetworkManager
144
+                add_udev_rule_to_unmanage_vf_representors_by_nm()
145
+
150 146
                 # Adding a udev rule to save the sriov_pf name
151
-                add_udev_rule_for_sriov_pf(item['name'])
152
-                add_udev_rule_for_vf_representors()
147
+                trigger_udev_rule = add_udev_rule_for_sriov_pf(item['name'])\
148
+                    or trigger_udev_rule
149
+
153 150
                 configure_switchdev(item['name'])
151
+
152
+                # Adding a udev rule to rename vf-representors
153
+                trigger_udev_rule = add_udev_rule_for_vf_representors(
154
+                    item['name']) or trigger_udev_rule
155
+
154 156
                 if_up_interface(item['name'])
155 157
 
158
+    # Trigger udev rules if there is new rules written
159
+    if trigger_udev_rule:
160
+        trigger_udev_rules()
161
+
156 162
     observer.stop()
157 163
 
158 164
 
@@ -184,6 +190,20 @@ def _wait_for_vf_creation(pf_name, numvfs):
184 190
     logger.info("Required VFs are created for PF %s" % pf_name)
185 191
 
186 192
 
193
+def _wait_for_uplink_rep_creation(pf_name):
194
+    uplink_rep_phys_switch_id_path = "/sys/class/net/%s/phys_switch_id" \
195
+                                     % pf_name
196
+
197
+    for i in range(MAX_RETRIES):
198
+        if get_file_data(uplink_rep_phys_switch_id_path):
199
+            logger.info("Uplink representor %s ready", pf_name)
200
+            break
201
+        time.sleep(1)
202
+    else:
203
+        raise RuntimeError("Timeout while waiting for uplink representor %s.",
204
+                           pf_name)
205
+
206
+
187 207
 def create_rep_link_name_script():
188 208
     with open(_REP_LINK_NAME_FILE, "w") as f:
189 209
         f.write(_REP_LINK_NAME_DATA)
@@ -195,25 +215,51 @@ def add_udev_rule_for_sriov_pf(pf_name):
195 215
     pf_pci = get_pf_pci(pf_name)
196 216
     udev_data_line = 'SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", '\
197 217
                      'KERNELS=="%s", NAME="%s"' % (pf_pci, pf_name)
198
-    add_udev_rule(udev_data_line, _UDEV_RULE_FILE)
218
+    return add_udev_rule(udev_data_line, _UDEV_RULE_FILE)
199 219
 
200 220
 
201
-def add_udev_rule_for_vf_representors():
221
+def add_udev_rule_for_vf_representors(pf_name):
222
+    phys_switch_id_path = os.path.join(_SYS_CLASS_NET, pf_name,
223
+                                       "phys_switch_id")
224
+    phys_switch_id = get_file_data(phys_switch_id_path).strip()
225
+    pf_pci = get_pf_pci(pf_name)
226
+    pf_fun_num_match = PF_FUNC_RE.search(pf_pci)
227
+    if pf_fun_num_match:
228
+        pf_fun_num = pf_fun_num_match.group(1)
229
+    else:
230
+        logger.error("Failed to get function number for %s \n"
231
+                     "and so failed to create a udev rule for renaming "
232
+                     "its' vf-represent" % pf_name)
233
+        return
234
+
202 235
     udev_data_line = 'SUBSYSTEM=="net", ACTION=="add", ATTR{phys_switch_id}'\
203
-                     '!="", ATTR{phys_port_name}=="pf*vf*", '\
204
-                     'IMPORT{program}="%s '\
205
-                     '$attr{phys_switch_id} $attr{phys_port_name}" '\
206
-                     'NAME="$env{NAME}"' % _REP_LINK_NAME_FILE
236
+                     '=="%s", ATTR{phys_port_name}=="pf%svf*", '\
237
+                     'IMPORT{program}="%s $attr{phys_port_name}", '\
238
+                     'NAME="%s_$env{NUMBER}"' % (phys_switch_id,
239
+                                                 pf_fun_num,
240
+                                                 _REP_LINK_NAME_FILE,
241
+                                                 pf_name)
207 242
     create_rep_link_name_script()
208
-    add_udev_rule(udev_data_line, _UDEV_RULE_FILE)
243
+    return add_udev_rule(udev_data_line, _UDEV_RULE_FILE)
244
+
245
+
246
+def add_udev_rule_to_unmanage_vf_representors_by_nm():
247
+    udev_data_line = 'SUBSYSTEM=="net", ACTION=="add", ATTR{phys_switch_id}'\
248
+                     '!="", ATTR{phys_port_name}=="pf*vf*", '\
249
+                     'ENV{NM_UNMANAGED}="1"'
250
+    return add_udev_rule(udev_data_line, _UDEV_RULE_FILE)
209 251
 
210 252
 
211 253
 def add_udev_rule(udev_data, udev_file):
254
+    trigger_udev_rule = False
212 255
     udev_data = udev_data.strip()
213 256
     if not os.path.exists(udev_file):
214 257
         with open(udev_file, "w") as f:
215
-            f.write(udev_data + "\n")
258
+            data = "# This file is autogenerated by os-net-config\n%s\n"\
259
+                   % udev_data
260
+            f.write(data)
216 261
         reload_udev_rules()
262
+        trigger_udev_rule = True
217 263
     else:
218 264
         file_data = get_file_data(udev_file)
219 265
         udev_lines = file_data.split("\n")
@@ -221,16 +267,29 @@ def add_udev_rule(udev_data, udev_file):
221 267
             with open(udev_file, "a") as f:
222 268
                 f.write(udev_data + "\n")
223 269
             reload_udev_rules()
270
+            trigger_udev_rule = True
271
+    return trigger_udev_rule
224 272
 
225 273
 
226 274
 def reload_udev_rules():
227 275
     try:
228 276
         processutils.execute('/usr/sbin/udevadm', 'control', '--reload-rules')
277
+        logger.info("udev rules reloaded successfully")
229 278
     except processutils.ProcessExecutionError:
230 279
         logger.error("Failed to reload udev rules")
231 280
         raise
232 281
 
233 282
 
283
+def trigger_udev_rules():
284
+    try:
285
+        processutils.execute('/usr/sbin/udevadm', 'trigger', '--action=add',
286
+                             '--attr-match=subsystem=net')
287
+        logger.info("udev rules triggered successfully")
288
+    except processutils.ProcessExecutionError:
289
+        logger.error("Failed to trigger udev rules")
290
+        raise
291
+
292
+
234 293
 def configure_switchdev(pf_name):
235 294
     pf_pci = get_pf_pci(pf_name)
236 295
     pf_device_id = get_pf_device_id(pf_name)
@@ -247,9 +306,18 @@ def configure_switchdev(pf_name):
247 306
     except processutils.ProcessExecutionError:
248 307
         logger.error("Failed to set mode to switchdev")
249 308
         raise
309
+    logger.info("Device pci/%s set to switchdev mode." % pf_pci)
310
+
311
+    # WA to make sure that the uplink_rep is ready after moving to switchdev,
312
+    # as moving to switchdev will remove the sriov_pf and create uplink
313
+    # representor, so we need to make sure that uplink representor is ready
314
+    # before proceed
315
+    _wait_for_uplink_rep_creation(pf_name)
316
+
250 317
     try:
251 318
         processutils.execute('/usr/sbin/ethtool', '-K', pf_name,
252 319
                              'hw-tc-offload', 'on')
320
+        logger.info("Enabled \"hw-tc-offload\" for PF %s." % pf_name)
253 321
     except processutils.ProcessExecutionError:
254 322
         logger.error("Failed to enable hw-tc-offload")
255 323
         raise

Loading…
Cancel
Save