diff --git a/os_net_config/sriov_bind_config.py b/os_net_config/sriov_bind_config.py new file mode 100644 index 00000000..90bc6cc8 --- /dev/null +++ b/os_net_config/sriov_bind_config.py @@ -0,0 +1,137 @@ +# Copyright 2020 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# + + +import logging +import os +import yaml + + +from oslo_concurrency import processutils + + +logger = logging.getLogger(__name__) +_PCI_DRIVER_BIND_FILE_PATH = "/sys/bus/pci/drivers/%(driver)s/bind" +_PCI_DRIVER_FILE_PATH = "/sys/bus/pci/devices/%(pci)s/driver" +_SRIOV_BIND_CONFIG_FILE = "/var/lib/os-net-config/sriov_bind_config.yaml" +_SRIOV_BIND_SERVICE_FILE = "/etc/systemd/system/sriov_bind.service" +_SRIOV_BIND_SERVICE_CONTENT = """[Unit] +Description=SR-IOV vfs binding +After=network.service sriov_config.service + +[Service] +Type=oneshot +ExecStart=/usr/bin/os-net-config-sriov-bind + +[Install] +WantedBy=multi-user.target +""" + +# File to contain the map of drivers and it's VFs list that should be bound +# Format of the file shall be +# : +# - '' +# - '' +# - '' +# - '' +# : +# - '' +# - '' +# - '' +# - '' + + +def get_file_data(filename): + if not os.path.exists(filename): + logger.error("Error file is not exist: %s" % filename) + raise FileNotFoundError(filename) + try: + with open(filename, 'r') as f: + return f.read() + except IOError: + logger.error("Error reading file: %s" % filename) + raise + + +def ensure_directory_presence(filepath): + dir_path = os.path.dirname(filepath) + if not os.path.exists(dir_path): + os.makedirs(dir_path) + + +def write_yaml_config(filepath, data): + ensure_directory_presence(filepath) + with open(filepath, 'w') as f: + yaml.safe_dump(data, f, default_flow_style=False) + + +def update_sriov_bind_pcis_map(sriov_bind_pcis_map): + sriov_bind_config_data = {} + try: + sriov_bind_config_data = _get_sriov_bind_pcis_map() + except Exception: + pass + # Compare two levels of the dictionary to conditionally write + # sriov_bind_pcis_map if it differs from existning sriov_bind_config_data + if (sriov_bind_config_data == {} or + set(sriov_bind_config_data.keys()) != + set(sriov_bind_pcis_map.keys()) or not + all([set(sriov_bind_config_data[key]) == + set(sriov_bind_pcis_map[key]) for key in + sriov_bind_config_data])): + write_yaml_config(_SRIOV_BIND_CONFIG_FILE, sriov_bind_pcis_map) + + +def _get_sriov_bind_pcis_map(): + contents = get_file_data(_SRIOV_BIND_CONFIG_FILE) + sriov_bind_pcis_map = yaml.safe_load(contents) if contents else {} + return sriov_bind_pcis_map + + +def configure_sriov_bind_service(): + """Generate the sriov_bind.service + + sriov_bind service shall bind all the vfs of switchdev-mode mlnx SriovPF + nics during reboot of the nodes. + """ + with open(_SRIOV_BIND_SERVICE_FILE, 'w') as f: + f.write(_SRIOV_BIND_SERVICE_CONTENT) + processutils.execute('systemctl', 'enable', 'sriov_bind') + + +def bind_vfs(sriov_bind_pcis_map=None): + if not sriov_bind_pcis_map: + sriov_bind_pcis_map = _get_sriov_bind_pcis_map() + for driver, pcis_list in sriov_bind_pcis_map.items(): + for vf_pci in pcis_list: + vf_pci_driver_path = _PCI_DRIVER_FILE_PATH % {"pci": vf_pci} + if not os.path.exists(vf_pci_driver_path): + pci_driver_bind_file_path = _PCI_DRIVER_BIND_FILE_PATH %\ + {"driver": driver} + try: + with open(pci_driver_bind_file_path, 'w') as f: + f.write("%s" % vf_pci) + logger.info("Vf %s has been bound" % vf_pci) + except IOError: + logger.error("Failed to bind vf %s" % vf_pci) + + +def main(): + bind_vfs() + + +if __name__ == "__main__": + main() diff --git a/os_net_config/sriov_config.py b/os_net_config/sriov_config.py index e3ccfe2b..3b98cb94 100644 --- a/os_net_config/sriov_config.py +++ b/os_net_config/sriov_config.py @@ -31,6 +31,7 @@ import sys import time import yaml +from os_net_config import sriov_bind_config from oslo_concurrency import processutils logger = logging.getLogger(__name__) @@ -40,6 +41,8 @@ _UDEV_LEGACY_RULE_FILE = '/etc/udev/rules.d/70-os-net-config-sriov.rules' _IFUP_LOCAL_FILE = '/sbin/ifup-local' _RESET_SRIOV_RULES_FILE = '/etc/udev/rules.d/70-tripleo-reset-sriov.rules' _ALLOCATE_VFS_FILE = '/etc/sysconfig/allocate_vfs' +_MLNX_DRIVER = "mlx5_core" +MLNX_VENDOR_ID = "0x15b3" MAX_RETRIES = 10 PF_FUNC_RE = re.compile(r"\.(\d+)$", 0) @@ -171,7 +174,7 @@ def configure_sriov_pf(execution_from_cli=False, restart_openvswitch=False): sriov_map = _get_sriov_map() MLNX_UNBIND_FILE_PATH = "/sys/bus/pci/drivers/mlx5_core/unbind" - MLNX_VENDOR_ID = "0x15b3" + mlnx_vfs_pcis_list = [] trigger_udev_rule = False # Cleanup the previous config by puppet-tripleo @@ -206,6 +209,7 @@ def configure_sriov_pf(execution_from_cli=False, restart_openvswitch=False): if (item.get('link_mode') == "switchdev" and vendor_id == MLNX_VENDOR_ID): vf_pcis_list = get_vf_pcis_list(item['name']) + mlnx_vfs_pcis_list += vf_pcis_list for vf_pci in vf_pcis_list: vf_pci_path = "/sys/bus/pci/devices/%s/driver" % vf_pci if os.path.exists(vf_pci_path): @@ -238,6 +242,14 @@ def configure_sriov_pf(execution_from_cli=False, restart_openvswitch=False): if execution_from_cli: if_up_interface(item['name']) + if mlnx_vfs_pcis_list: + sriov_bind_pcis_map = {_MLNX_DRIVER: mlnx_vfs_pcis_list} + if not execution_from_cli: + sriov_bind_config.update_sriov_bind_pcis_map(sriov_bind_pcis_map) + else: + sriov_bind_config.configure_sriov_bind_service() + sriov_bind_config.bind_vfs(sriov_bind_pcis_map) + # Trigger udev rules if there is new rules written if trigger_udev_rule: trigger_udev_rules() diff --git a/os_net_config/tests/test_sriov_bind_config.py b/os_net_config/tests/test_sriov_bind_config.py new file mode 100644 index 00000000..aad9e605 --- /dev/null +++ b/os_net_config/tests/test_sriov_bind_config.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import os.path +import random + + +from os_net_config import sriov_bind_config +from os_net_config.tests import base +from os_net_config import utils + + +class TestSriovBindConfig(base.TestCase): + """Unit tests for methods defined in sriov_bind_config.py""" + + def setUp(self): + super(TestSriovBindConfig, self).setUp() + rand = str(int(random.random() * 100000)) + + sriov_bind_config._SRIOV_BIND_CONFIG_FILE = '/tmp/' + rand +\ + 'sriov_bind_config.yaml' + sriov_bind_config._PCI_DRIVER_BIND_FILE_PATH = '/tmp/' + rand +\ + '%(driver)s/bind' + + def tearDown(self): + super(TestSriovBindConfig, self).tearDown() + if os.path.isfile(sriov_bind_config._SRIOV_BIND_CONFIG_FILE): + os.remove(sriov_bind_config._SRIOV_BIND_CONFIG_FILE) + + def test_bind_vfs(self): + """Test SR-IOV VFs binding""" + vfs_driver = "mlx5_core" + sriov_bind_pcis_map = {vfs_driver: ['0000:03:00.2', '0000:03:00.3']} + os.makedirs(sriov_bind_config._PCI_DRIVER_BIND_FILE_PATH % + {"driver": vfs_driver}) + + utils.write_yaml_config(sriov_bind_config._SRIOV_BIND_CONFIG_FILE, + sriov_bind_pcis_map) + sriov_bind_config.bind_vfs() diff --git a/setup.cfg b/setup.cfg index 02f93734..dc6a6a68 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,3 +30,4 @@ packages = console_scripts = os-net-config = os_net_config.cli:main os-net-config-sriov = os_net_config.sriov_config:main + os-net-config-sriov-bind = os_net_config.sriov_bind_config:main