openstack
/
os-net-config
Code Issues Proposed changes

Restructuring the method of vf-representor renaming 

The old way of renmaing depended on adding udev rule
before creating vf-representors and so we faced race condition on
uplink representor rename.
So now we are creating vf-representors and after that we adding the
udev rules and trigerring them at the end
Also because the old name of UDEV_RULE_FILE(70-persistent-net.rules) is
used by another module, so we renamed the udev_rule_file to
(80-persistent-os-net-config.rules) and with that way it will override
any previouse rules in (70-persistent-net.rules) and will not any cause
any confustion
Addition to that, we also added a udev rule to make vf-representors
unmanaged by NetworkManager in order to be renamed successfully

Change-Id: I61c59e162bc5d15c063f52612e198d93f4433db1
Closes-Bug: #1830716
(cherry picked from commit 5d44b582b5)
This commit is contained in:
waleed mousa 2019-06-13 06:33:29 -04:00 committed by Saravanan KR
parent 6d7e74a381
commit bab5da768b
1 changed files with 97 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

126
os_net_config/sriov_config.py
View File

@ -28,36 +28,26 @@ import pyudev
import re
from six.moves import queue as Queue
import sys
import time
import yaml
from oslo_concurrency import processutils
logger = logging.getLogger(__name__)
_SYS_CLASS_NET = '/sys/class/net'
_UDEV_RULE_FILE = '/etc/udev/rules.d/70-persistent-net.rules'
_UDEV_RULE_FILE = '/etc/udev/rules.d/80-persistent-os-net-config.rules'
MAX_RETRIES = 10
PF_FUNC_RE = re.compile(r"\.(\d+)$", 0)
# In order to keep VF representor name consistent specially after the upgrade
# proccess, we should have a udev rule to handle that.
# The udev rule will rename the VF representor as "<sriov_pf_name>_<vf_num>"
_REP_LINK_NAME_FILE = "/etc/udev/rep-link-name.sh"
_REP_LINK_NAME_DATA = '''#!/bin/bash
SWID="$1"
PORT="$2"
parent_phys_port_name=${PORT%vf*}
parent_phys_port_name=${parent_phys_port_name//f}
for i in `ls -1 /sys/class/net/*/phys_port_name`
do
nic=`echo $i | cut -d/ -f 5`
sw_id=`cat /sys/class/net/$nic/phys_switch_id 2>/dev/null`
phys_port_name=`cat /sys/class/net/$nic/phys_port_name 2>/dev/null`
if [ "$parent_phys_port_name" = "$phys_port_name" ] &&
[ "$sw_id" = "$SWID" ]
then
echo "NAME=${nic}_${PORT##pf*vf}"
break
exit
fi
done'''
# This file is autogenerated by os-net-config
set -x
PORT="$1"
echo "NUMBER=${PORT##pf*vf}"
'''
# Create a queue for passing the udev network events
vf_queue = Queue.Queue()
@ -123,6 +113,7 @@ def configure_sriov_pf():
sriov_map = _get_sriov_map()
MLNX_UNBIND_FILE_PATH = "/sys/bus/pci/drivers/mlx5_core/unbind"
MLNX_VENDOR_ID = "0x15b3"
trigger_udev_rule = False
for item in sriov_map:
if item['device_type'] == 'pf':
_pf_interface_up(item)
@ -147,12 +138,27 @@ def configure_sriov_pf():
if os.path.exists(vf_pci_path):
with open(MLNX_UNBIND_FILE_PATH, 'w') as f:
f.write("%s" % vf_pci)
# Adding a udev rule to make vf-representors unmanaged by
# NetworkManager
add_udev_rule_to_unmanage_vf_representors_by_nm()
# Adding a udev rule to save the sriov_pf name
add_udev_rule_for_sriov_pf(item['name'])
add_udev_rule_for_vf_representors()
trigger_udev_rule = add_udev_rule_for_sriov_pf(item['name'])\
or trigger_udev_rule
configure_switchdev(item['name'])
# Adding a udev rule to rename vf-representors
trigger_udev_rule = add_udev_rule_for_vf_representors(
item['name']) or trigger_udev_rule
if_up_interface(item['name'])
# Trigger udev rules if there is new rules written
if trigger_udev_rule:
trigger_udev_rules()
observer.stop()
@ -184,6 +190,20 @@ def _wait_for_vf_creation(pf_name, numvfs):
logger.info("Required VFs are created for PF %s" % pf_name)
def _wait_for_uplink_rep_creation(pf_name):
uplink_rep_phys_switch_id_path = "/sys/class/net/%s/phys_switch_id" \
% pf_name
for i in range(MAX_RETRIES):
if get_file_data(uplink_rep_phys_switch_id_path):
logger.info("Uplink representor %s ready", pf_name)
break
time.sleep(1)
else:
raise RuntimeError("Timeout while waiting for uplink representor %s.",
pf_name)
def create_rep_link_name_script():
with open(_REP_LINK_NAME_FILE, "w") as f:
f.write(_REP_LINK_NAME_DATA)
@ -195,25 +215,51 @@ def add_udev_rule_for_sriov_pf(pf_name):
pf_pci = get_pf_pci(pf_name)
udev_data_line = 'SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", '\
'KERNELS=="%s", NAME="%s"' % (pf_pci, pf_name)
add_udev_rule(udev_data_line, _UDEV_RULE_FILE)
return add_udev_rule(udev_data_line, _UDEV_RULE_FILE)
def add_udev_rule_for_vf_representors():
def add_udev_rule_for_vf_representors(pf_name):
phys_switch_id_path = os.path.join(_SYS_CLASS_NET, pf_name,
"phys_switch_id")
phys_switch_id = get_file_data(phys_switch_id_path).strip()
pf_pci = get_pf_pci(pf_name)
pf_fun_num_match = PF_FUNC_RE.search(pf_pci)
if pf_fun_num_match:
pf_fun_num = pf_fun_num_match.group(1)
else:
logger.error("Failed to get function number for %s \n"
"and so failed to create a udev rule for renaming "
"its' vf-represent" % pf_name)
return
udev_data_line = 'SUBSYSTEM=="net", ACTION=="add", ATTR{phys_switch_id}'\
'=="%s", ATTR{phys_port_name}=="pf%svf*", '\
'IMPORT{program}="%s $attr{phys_port_name}", '\
'NAME="%s_$env{NUMBER}"' % (phys_switch_id,
pf_fun_num,
_REP_LINK_NAME_FILE,
pf_name)
create_rep_link_name_script()
return add_udev_rule(udev_data_line, _UDEV_RULE_FILE)
def add_udev_rule_to_unmanage_vf_representors_by_nm():
udev_data_line = 'SUBSYSTEM=="net", ACTION=="add", ATTR{phys_switch_id}'\
'!="", ATTR{phys_port_name}=="pf*vf*", '\
'IMPORT{program}="%s '\
'$attr{phys_switch_id} $attr{phys_port_name}" '\
'NAME="$env{NAME}"' % _REP_LINK_NAME_FILE
create_rep_link_name_script()
add_udev_rule(udev_data_line, _UDEV_RULE_FILE)
'ENV{NM_UNMANAGED}="1"'
return add_udev_rule(udev_data_line, _UDEV_RULE_FILE)
def add_udev_rule(udev_data, udev_file):
trigger_udev_rule = False
udev_data = udev_data.strip()
if not os.path.exists(udev_file):
with open(udev_file, "w") as f:
f.write(udev_data + "\n")
data = "# This file is autogenerated by os-net-config\n%s\n"\
% udev_data
f.write(data)
reload_udev_rules()
trigger_udev_rule = True
else:
file_data = get_file_data(udev_file)
udev_lines = file_data.split("\n")
@ -221,16 +267,29 @@ def add_udev_rule(udev_data, udev_file):
with open(udev_file, "a") as f:
f.write(udev_data + "\n")
reload_udev_rules()
trigger_udev_rule = True
return trigger_udev_rule
def reload_udev_rules():
try:
processutils.execute('/usr/sbin/udevadm', 'control', '--reload-rules')
logger.info("udev rules reloaded successfully")
except processutils.ProcessExecutionError:
logger.error("Failed to reload udev rules")
raise
def trigger_udev_rules():
try:
processutils.execute('/usr/sbin/udevadm', 'trigger', '--action=add',
'--attr-match=subsystem=net')
logger.info("udev rules triggered successfully")
except processutils.ProcessExecutionError:
logger.error("Failed to trigger udev rules")
raise
def configure_switchdev(pf_name):
pf_pci = get_pf_pci(pf_name)
pf_device_id = get_pf_device_id(pf_name)
@ -247,9 +306,18 @@ def configure_switchdev(pf_name):
except processutils.ProcessExecutionError:
logger.error("Failed to set mode to switchdev")
raise
logger.info("Device pci/%s set to switchdev mode." % pf_pci)
# WA to make sure that the uplink_rep is ready after moving to switchdev,
# as moving to switchdev will remove the sriov_pf and create uplink
# representor, so we need to make sure that uplink representor is ready
# before proceed
_wait_for_uplink_rep_creation(pf_name)
try:
processutils.execute('/usr/sbin/ethtool', '-K', pf_name,
'hw-tc-offload', 'on')
logger.info("Enabled \"hw-tc-offload\" for PF %s." % pf_name)
except processutils.ProcessExecutionError:
logger.error("Failed to enable hw-tc-offload")
raise