Support binding VFs after moving to switchdev
Currently for Mellanox vendor ID we don't bind vfs after moving to switchdev
because it affects VF-lAG feature, but for other features like OVS-DPDK SRIOV,
OVS-DPDK VDPA and OVS-Kernel Forwarder we have to bind them.
To support binding VFs to networking driver, there are two scenarios
that we need to address:
* Deployment case
In deployment case, we are binding vfs after moving all the
sriov_pfs to switchdev and also configuring sriov_bind service
* Reboot case
In reboot case, we start sriov_bind service which will run
after network.service sriov_config.service (The case that we are
sure that VF-LAG activation done)
Change-Id: I7e79029602f403885b347289d2ae6a3d47453a4e
(cherry picked from commit 3144a8fc4f
)
This commit is contained in:
parent
24906635ca
commit
157df251cf
|
@ -0,0 +1,137 @@
|
|||
# Copyright 2020 Red Hat, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
#
|
||||
|
||||
|
||||
import logging
|
||||
import os
|
||||
import yaml
|
||||
|
||||
|
||||
from oslo_concurrency import processutils
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_PCI_DRIVER_BIND_FILE_PATH = "/sys/bus/pci/drivers/%(driver)s/bind"
|
||||
_PCI_DRIVER_FILE_PATH = "/sys/bus/pci/devices/%(pci)s/driver"
|
||||
_SRIOV_BIND_CONFIG_FILE = "/var/lib/os-net-config/sriov_bind_config.yaml"
|
||||
_SRIOV_BIND_SERVICE_FILE = "/etc/systemd/system/sriov_bind.service"
|
||||
_SRIOV_BIND_SERVICE_CONTENT = """[Unit]
|
||||
Description=SR-IOV vfs binding
|
||||
After=network.service sriov_config.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/os-net-config-sriov-bind
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
"""
|
||||
|
||||
# File to contain the map of drivers and it's VFs list that should be bound
|
||||
# Format of the file shall be
|
||||
# <driver1>:
|
||||
# - '<VF1_PCI>'
|
||||
# - '<VF2_PCI>'
|
||||
# - '<VF3_PCI>'
|
||||
# - '<VF4_PCI>'
|
||||
# <driver2>:
|
||||
# - '<VF5_PCI>'
|
||||
# - '<VF6_PCI>'
|
||||
# - '<VF7_PCI>'
|
||||
# - '<VF8_PCI>'
|
||||
|
||||
|
||||
def get_file_data(filename):
|
||||
if not os.path.exists(filename):
|
||||
logger.error("Error file is not exist: %s" % filename)
|
||||
raise FileNotFoundError(filename)
|
||||
try:
|
||||
with open(filename, 'r') as f:
|
||||
return f.read()
|
||||
except IOError:
|
||||
logger.error("Error reading file: %s" % filename)
|
||||
raise
|
||||
|
||||
|
||||
def ensure_directory_presence(filepath):
|
||||
dir_path = os.path.dirname(filepath)
|
||||
if not os.path.exists(dir_path):
|
||||
os.makedirs(dir_path)
|
||||
|
||||
|
||||
def write_yaml_config(filepath, data):
|
||||
ensure_directory_presence(filepath)
|
||||
with open(filepath, 'w') as f:
|
||||
yaml.safe_dump(data, f, default_flow_style=False)
|
||||
|
||||
|
||||
def update_sriov_bind_pcis_map(sriov_bind_pcis_map):
|
||||
sriov_bind_config_data = {}
|
||||
try:
|
||||
sriov_bind_config_data = _get_sriov_bind_pcis_map()
|
||||
except Exception:
|
||||
pass
|
||||
# Compare two levels of the dictionary to conditionally write
|
||||
# sriov_bind_pcis_map if it differs from existning sriov_bind_config_data
|
||||
if (sriov_bind_config_data == {} or
|
||||
set(sriov_bind_config_data.keys()) !=
|
||||
set(sriov_bind_pcis_map.keys()) or not
|
||||
all([set(sriov_bind_config_data[key]) ==
|
||||
set(sriov_bind_pcis_map[key]) for key in
|
||||
sriov_bind_config_data])):
|
||||
write_yaml_config(_SRIOV_BIND_CONFIG_FILE, sriov_bind_pcis_map)
|
||||
|
||||
|
||||
def _get_sriov_bind_pcis_map():
|
||||
contents = get_file_data(_SRIOV_BIND_CONFIG_FILE)
|
||||
sriov_bind_pcis_map = yaml.safe_load(contents) if contents else {}
|
||||
return sriov_bind_pcis_map
|
||||
|
||||
|
||||
def configure_sriov_bind_service():
|
||||
"""Generate the sriov_bind.service
|
||||
|
||||
sriov_bind service shall bind all the vfs of switchdev-mode mlnx SriovPF
|
||||
nics during reboot of the nodes.
|
||||
"""
|
||||
with open(_SRIOV_BIND_SERVICE_FILE, 'w') as f:
|
||||
f.write(_SRIOV_BIND_SERVICE_CONTENT)
|
||||
processutils.execute('systemctl', 'enable', 'sriov_bind')
|
||||
|
||||
|
||||
def bind_vfs(sriov_bind_pcis_map=None):
|
||||
if not sriov_bind_pcis_map:
|
||||
sriov_bind_pcis_map = _get_sriov_bind_pcis_map()
|
||||
for driver, pcis_list in sriov_bind_pcis_map.items():
|
||||
for vf_pci in pcis_list:
|
||||
vf_pci_driver_path = _PCI_DRIVER_FILE_PATH % {"pci": vf_pci}
|
||||
if not os.path.exists(vf_pci_driver_path):
|
||||
pci_driver_bind_file_path = _PCI_DRIVER_BIND_FILE_PATH %\
|
||||
{"driver": driver}
|
||||
try:
|
||||
with open(pci_driver_bind_file_path, 'w') as f:
|
||||
f.write("%s" % vf_pci)
|
||||
logger.info("Vf %s has been bound" % vf_pci)
|
||||
except IOError:
|
||||
logger.error("Failed to bind vf %s" % vf_pci)
|
||||
|
||||
|
||||
def main():
|
||||
bind_vfs()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
|
@ -31,6 +31,7 @@ import sys
|
|||
import time
|
||||
import yaml
|
||||
|
||||
from os_net_config import sriov_bind_config
|
||||
from oslo_concurrency import processutils
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
@ -40,6 +41,8 @@ _UDEV_LEGACY_RULE_FILE = '/etc/udev/rules.d/70-os-net-config-sriov.rules'
|
|||
_IFUP_LOCAL_FILE = '/sbin/ifup-local'
|
||||
_RESET_SRIOV_RULES_FILE = '/etc/udev/rules.d/70-tripleo-reset-sriov.rules'
|
||||
_ALLOCATE_VFS_FILE = '/etc/sysconfig/allocate_vfs'
|
||||
_MLNX_DRIVER = "mlx5_core"
|
||||
MLNX_VENDOR_ID = "0x15b3"
|
||||
|
||||
MAX_RETRIES = 10
|
||||
PF_FUNC_RE = re.compile(r"\.(\d+)$", 0)
|
||||
|
@ -171,7 +174,7 @@ def configure_sriov_pf(execution_from_cli=False, restart_openvswitch=False):
|
|||
|
||||
sriov_map = _get_sriov_map()
|
||||
MLNX_UNBIND_FILE_PATH = "/sys/bus/pci/drivers/mlx5_core/unbind"
|
||||
MLNX_VENDOR_ID = "0x15b3"
|
||||
mlnx_vfs_pcis_list = []
|
||||
trigger_udev_rule = False
|
||||
|
||||
# Cleanup the previous config by puppet-tripleo
|
||||
|
@ -206,6 +209,7 @@ def configure_sriov_pf(execution_from_cli=False, restart_openvswitch=False):
|
|||
if (item.get('link_mode') == "switchdev" and
|
||||
vendor_id == MLNX_VENDOR_ID):
|
||||
vf_pcis_list = get_vf_pcis_list(item['name'])
|
||||
mlnx_vfs_pcis_list += vf_pcis_list
|
||||
for vf_pci in vf_pcis_list:
|
||||
vf_pci_path = "/sys/bus/pci/devices/%s/driver" % vf_pci
|
||||
if os.path.exists(vf_pci_path):
|
||||
|
@ -238,6 +242,14 @@ def configure_sriov_pf(execution_from_cli=False, restart_openvswitch=False):
|
|||
if execution_from_cli:
|
||||
if_up_interface(item['name'])
|
||||
|
||||
if mlnx_vfs_pcis_list:
|
||||
sriov_bind_pcis_map = {_MLNX_DRIVER: mlnx_vfs_pcis_list}
|
||||
if not execution_from_cli:
|
||||
sriov_bind_config.update_sriov_bind_pcis_map(sriov_bind_pcis_map)
|
||||
else:
|
||||
sriov_bind_config.configure_sriov_bind_service()
|
||||
sriov_bind_config.bind_vfs(sriov_bind_pcis_map)
|
||||
|
||||
# Trigger udev rules if there is new rules written
|
||||
if trigger_udev_rule:
|
||||
trigger_udev_rules()
|
||||
|
|
|
@ -0,0 +1,53 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2019 Red Hat, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import os
|
||||
import os.path
|
||||
import random
|
||||
|
||||
|
||||
from os_net_config import sriov_bind_config
|
||||
from os_net_config.tests import base
|
||||
from os_net_config import utils
|
||||
|
||||
|
||||
class TestSriovBindConfig(base.TestCase):
|
||||
"""Unit tests for methods defined in sriov_bind_config.py"""
|
||||
|
||||
def setUp(self):
|
||||
super(TestSriovBindConfig, self).setUp()
|
||||
rand = str(int(random.random() * 100000))
|
||||
|
||||
sriov_bind_config._SRIOV_BIND_CONFIG_FILE = '/tmp/' + rand +\
|
||||
'sriov_bind_config.yaml'
|
||||
sriov_bind_config._PCI_DRIVER_BIND_FILE_PATH = '/tmp/' + rand +\
|
||||
'%(driver)s/bind'
|
||||
|
||||
def tearDown(self):
|
||||
super(TestSriovBindConfig, self).tearDown()
|
||||
if os.path.isfile(sriov_bind_config._SRIOV_BIND_CONFIG_FILE):
|
||||
os.remove(sriov_bind_config._SRIOV_BIND_CONFIG_FILE)
|
||||
|
||||
def test_bind_vfs(self):
|
||||
"""Test SR-IOV VFs binding"""
|
||||
vfs_driver = "mlx5_core"
|
||||
sriov_bind_pcis_map = {vfs_driver: ['0000:03:00.2', '0000:03:00.3']}
|
||||
os.makedirs(sriov_bind_config._PCI_DRIVER_BIND_FILE_PATH %
|
||||
{"driver": vfs_driver})
|
||||
|
||||
utils.write_yaml_config(sriov_bind_config._SRIOV_BIND_CONFIG_FILE,
|
||||
sriov_bind_pcis_map)
|
||||
sriov_bind_config.bind_vfs()
|
Loading…
Reference in New Issue