Browse Source

Support binding VFs after moving to switchdev

Currently for Mellanox vendor ID we don't bind vfs after moving to switchdev
because it affects VF-lAG feature, but for other features like OVS-DPDK SRIOV,
OVS-DPDK VDPA and OVS-Kernel Forwarder we have to bind them.

To support binding VFs to networking driver, there are two scenarios
that we need to address:
    * Deployment case
      In deployment case, we are binding vfs after moving all the
      sriov_pfs to switchdev and also configuring sriov_bind service
    * Reboot case
      In reboot case, we start sriov_bind service which will run
      after network.service sriov_config.service (The case that we are
      sure that VF-LAG activation done)

Change-Id: I7e79029602f403885b347289d2ae6a3d47453a4e
changes/06/750206/13
waleedm 5 months ago
committed by waleed mousa
parent
commit
3144a8fc4f
4 changed files with 204 additions and 1 deletions
  1. +137
    -0
      os_net_config/sriov_bind_config.py
  2. +13
    -1
      os_net_config/sriov_config.py
  3. +53
    -0
      os_net_config/tests/test_sriov_bind_config.py
  4. +1
    -0
      setup.cfg

+ 137
- 0
os_net_config/sriov_bind_config.py View File

@ -0,0 +1,137 @@
# Copyright 2020 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import logging
import os
import yaml
from oslo_concurrency import processutils
logger = logging.getLogger(__name__)
_PCI_DRIVER_BIND_FILE_PATH = "/sys/bus/pci/drivers/%(driver)s/bind"
_PCI_DRIVER_FILE_PATH = "/sys/bus/pci/devices/%(pci)s/driver"
_SRIOV_BIND_CONFIG_FILE = "/var/lib/os-net-config/sriov_bind_config.yaml"
_SRIOV_BIND_SERVICE_FILE = "/etc/systemd/system/sriov_bind.service"
_SRIOV_BIND_SERVICE_CONTENT = """[Unit]
Description=SR-IOV vfs binding
After=network.service sriov_config.service
[Service]
Type=oneshot
ExecStart=/usr/bin/os-net-config-sriov-bind
[Install]
WantedBy=multi-user.target
"""
# File to contain the map of drivers and it's VFs list that should be bound
# Format of the file shall be
# <driver1>:
# - '<VF1_PCI>'
# - '<VF2_PCI>'
# - '<VF3_PCI>'
# - '<VF4_PCI>'
# <driver2>:
# - '<VF5_PCI>'
# - '<VF6_PCI>'
# - '<VF7_PCI>'
# - '<VF8_PCI>'
def get_file_data(filename):
if not os.path.exists(filename):
logger.error("Error file is not exist: %s" % filename)
raise FileNotFoundError(filename)
try:
with open(filename, 'r') as f:
return f.read()
except IOError:
logger.error("Error reading file: %s" % filename)
raise
def ensure_directory_presence(filepath):
dir_path = os.path.dirname(filepath)
if not os.path.exists(dir_path):
os.makedirs(dir_path)
def write_yaml_config(filepath, data):
ensure_directory_presence(filepath)
with open(filepath, 'w') as f:
yaml.safe_dump(data, f, default_flow_style=False)
def update_sriov_bind_pcis_map(sriov_bind_pcis_map):
sriov_bind_config_data = {}
try:
sriov_bind_config_data = _get_sriov_bind_pcis_map()
except Exception:
pass
# Compare two levels of the dictionary to conditionally write
# sriov_bind_pcis_map if it differs from existning sriov_bind_config_data
if (sriov_bind_config_data == {} or
set(sriov_bind_config_data.keys()) !=
set(sriov_bind_pcis_map.keys()) or not
all([set(sriov_bind_config_data[key]) ==
set(sriov_bind_pcis_map[key]) for key in
sriov_bind_config_data])):
write_yaml_config(_SRIOV_BIND_CONFIG_FILE, sriov_bind_pcis_map)
def _get_sriov_bind_pcis_map():
contents = get_file_data(_SRIOV_BIND_CONFIG_FILE)
sriov_bind_pcis_map = yaml.safe_load(contents) if contents else {}
return sriov_bind_pcis_map
def configure_sriov_bind_service():
"""Generate the sriov_bind.service
sriov_bind service shall bind all the vfs of switchdev-mode mlnx SriovPF
nics during reboot of the nodes.
"""
with open(_SRIOV_BIND_SERVICE_FILE, 'w') as f:
f.write(_SRIOV_BIND_SERVICE_CONTENT)
processutils.execute('systemctl', 'enable', 'sriov_bind')
def bind_vfs(sriov_bind_pcis_map=None):
if not sriov_bind_pcis_map:
sriov_bind_pcis_map = _get_sriov_bind_pcis_map()
for driver, pcis_list in sriov_bind_pcis_map.items():
for vf_pci in pcis_list:
vf_pci_driver_path = _PCI_DRIVER_FILE_PATH % {"pci": vf_pci}
if not os.path.exists(vf_pci_driver_path):
pci_driver_bind_file_path = _PCI_DRIVER_BIND_FILE_PATH %\
{"driver": driver}
try:
with open(pci_driver_bind_file_path, 'w') as f:
f.write("%s" % vf_pci)
logger.info("Vf %s has been bound" % vf_pci)
except IOError:
logger.error("Failed to bind vf %s" % vf_pci)
def main():
bind_vfs()
if __name__ == "__main__":
main()

+ 13
- 1
os_net_config/sriov_config.py View File

@ -31,6 +31,7 @@ import sys
import time
import yaml
from os_net_config import sriov_bind_config
from oslo_concurrency import processutils
logger = logging.getLogger(__name__)
@ -40,6 +41,8 @@ _UDEV_LEGACY_RULE_FILE = '/etc/udev/rules.d/70-os-net-config-sriov.rules'
_IFUP_LOCAL_FILE = '/sbin/ifup-local'
_RESET_SRIOV_RULES_FILE = '/etc/udev/rules.d/70-tripleo-reset-sriov.rules'
_ALLOCATE_VFS_FILE = '/etc/sysconfig/allocate_vfs'
_MLNX_DRIVER = "mlx5_core"
MLNX_VENDOR_ID = "0x15b3"
MAX_RETRIES = 10
PF_FUNC_RE = re.compile(r"\.(\d+)$", 0)
@ -171,7 +174,7 @@ def configure_sriov_pf(execution_from_cli=False, restart_openvswitch=False):
sriov_map = _get_sriov_map()
MLNX_UNBIND_FILE_PATH = "/sys/bus/pci/drivers/mlx5_core/unbind"
MLNX_VENDOR_ID = "0x15b3"
mlnx_vfs_pcis_list = []
trigger_udev_rule = False
# Cleanup the previous config by puppet-tripleo
@ -206,6 +209,7 @@ def configure_sriov_pf(execution_from_cli=False, restart_openvswitch=False):
if (item.get('link_mode') == "switchdev" and
vendor_id == MLNX_VENDOR_ID):
vf_pcis_list = get_vf_pcis_list(item['name'])
mlnx_vfs_pcis_list += vf_pcis_list
for vf_pci in vf_pcis_list:
vf_pci_path = "/sys/bus/pci/devices/%s/driver" % vf_pci
if os.path.exists(vf_pci_path):
@ -238,6 +242,14 @@ def configure_sriov_pf(execution_from_cli=False, restart_openvswitch=False):
if execution_from_cli:
if_up_interface(item['name'])
if mlnx_vfs_pcis_list:
sriov_bind_pcis_map = {_MLNX_DRIVER: mlnx_vfs_pcis_list}
if not execution_from_cli:
sriov_bind_config.update_sriov_bind_pcis_map(sriov_bind_pcis_map)
else:
sriov_bind_config.configure_sriov_bind_service()
sriov_bind_config.bind_vfs(sriov_bind_pcis_map)
# Trigger udev rules if there is new rules written
if trigger_udev_rule:
trigger_udev_rules()


+ 53
- 0
os_net_config/tests/test_sriov_bind_config.py View File

@ -0,0 +1,53 @@
# -*- coding: utf-8 -*-
# Copyright 2019 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import os.path
import random
from os_net_config import sriov_bind_config
from os_net_config.tests import base
from os_net_config import utils
class TestSriovBindConfig(base.TestCase):
"""Unit tests for methods defined in sriov_bind_config.py"""
def setUp(self):
super(TestSriovBindConfig, self).setUp()
rand = str(int(random.random() * 100000))
sriov_bind_config._SRIOV_BIND_CONFIG_FILE = '/tmp/' + rand +\
'sriov_bind_config.yaml'
sriov_bind_config._PCI_DRIVER_BIND_FILE_PATH = '/tmp/' + rand +\
'%(driver)s/bind'
def tearDown(self):
super(TestSriovBindConfig, self).tearDown()
if os.path.isfile(sriov_bind_config._SRIOV_BIND_CONFIG_FILE):
os.remove(sriov_bind_config._SRIOV_BIND_CONFIG_FILE)
def test_bind_vfs(self):
"""Test SR-IOV VFs binding"""
vfs_driver = "mlx5_core"
sriov_bind_pcis_map = {vfs_driver: ['0000:03:00.2', '0000:03:00.3']}
os.makedirs(sriov_bind_config._PCI_DRIVER_BIND_FILE_PATH %
{"driver": vfs_driver})
utils.write_yaml_config(sriov_bind_config._SRIOV_BIND_CONFIG_FILE,
sriov_bind_pcis_map)
sriov_bind_config.bind_vfs()

+ 1
- 0
setup.cfg View File

@ -30,3 +30,4 @@ packages =
console_scripts =
os-net-config = os_net_config.cli:main
os-net-config-sriov = os_net_config.sriov_config:main
os-net-config-sriov-bind = os_net_config.sriov_bind_config:main

Loading…
Cancel
Save