Support binding VFs after moving to switchdev

Currently for Mellanox vendor ID we don't bind vfs after moving to switchdev
because it affects VF-lAG feature, but for other features like OVS-DPDK SRIOV,
OVS-DPDK VDPA and OVS-Kernel Forwarder we have to bind them.

To support binding VFs to networking driver, there are two scenarios
that we need to address:
    * Deployment case
      In deployment case, we are binding vfs after moving all the
      sriov_pfs to switchdev and also configuring sriov_bind service
    * Reboot case
      In reboot case, we start sriov_bind service which will run
      after network.service sriov_config.service (The case that we are
      sure that VF-LAG activation done)

Change-Id: I7e79029602f403885b347289d2ae6a3d47453a4e

os_net_config/sriov_bind_config.py

@@ -0,0 +1,137 @@
+# Copyright 2020 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+#
+
+
+import logging
+import os
+import yaml
+
+
+from oslo_concurrency import processutils
+
+
+logger = logging.getLogger(__name__)
+_PCI_DRIVER_BIND_FILE_PATH = "/sys/bus/pci/drivers/%(driver)s/bind"
+_PCI_DRIVER_FILE_PATH = "/sys/bus/pci/devices/%(pci)s/driver"
+_SRIOV_BIND_CONFIG_FILE = "/var/lib/os-net-config/sriov_bind_config.yaml"
+_SRIOV_BIND_SERVICE_FILE = "/etc/systemd/system/sriov_bind.service"
+_SRIOV_BIND_SERVICE_CONTENT = """[Unit]
+Description=SR-IOV vfs binding
+After=network.service sriov_config.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/os-net-config-sriov-bind
+
+[Install]
+WantedBy=multi-user.target
+"""
+
+# File to contain the map of drivers and it's VFs list that should be bound
+# Format of the file shall be
+# <driver1>:
+# - '<VF1_PCI>'
+# - '<VF2_PCI>'
+# - '<VF3_PCI>'
+# - '<VF4_PCI>'
+# <driver2>:
+# - '<VF5_PCI>'
+# - '<VF6_PCI>'
+# - '<VF7_PCI>'
+# - '<VF8_PCI>'
+
+
+def get_file_data(filename):
+    if not os.path.exists(filename):
+        logger.error("Error file is not exist: %s" % filename)
+        raise FileNotFoundError(filename)
+    try:
+        with open(filename, 'r') as f:
+            return f.read()
+    except IOError:
+        logger.error("Error reading file: %s" % filename)
+        raise
+
+
+def ensure_directory_presence(filepath):
+    dir_path = os.path.dirname(filepath)
+    if not os.path.exists(dir_path):
+        os.makedirs(dir_path)
+
+
+def write_yaml_config(filepath, data):
+    ensure_directory_presence(filepath)
+    with open(filepath, 'w') as f:
+        yaml.safe_dump(data, f, default_flow_style=False)
+
+
+def update_sriov_bind_pcis_map(sriov_bind_pcis_map):
+    sriov_bind_config_data = {}
+    try:
+        sriov_bind_config_data = _get_sriov_bind_pcis_map()
+    except Exception:
+        pass
+    # Compare two levels of the dictionary to conditionally write
+    # sriov_bind_pcis_map if it differs from existning sriov_bind_config_data
+    if (sriov_bind_config_data == {} or
+            set(sriov_bind_config_data.keys()) !=
+            set(sriov_bind_pcis_map.keys()) or not
+            all([set(sriov_bind_config_data[key]) ==
+                 set(sriov_bind_pcis_map[key]) for key in
+                 sriov_bind_config_data])):
+        write_yaml_config(_SRIOV_BIND_CONFIG_FILE, sriov_bind_pcis_map)
+
+
+def _get_sriov_bind_pcis_map():
+    contents = get_file_data(_SRIOV_BIND_CONFIG_FILE)
+    sriov_bind_pcis_map = yaml.safe_load(contents) if contents else {}
+    return sriov_bind_pcis_map
+
+
+def configure_sriov_bind_service():
+    """Generate the sriov_bind.service
+
+       sriov_bind service shall bind all the vfs of switchdev-mode mlnx SriovPF
+       nics during reboot of the nodes.
+    """
+    with open(_SRIOV_BIND_SERVICE_FILE, 'w') as f:
+        f.write(_SRIOV_BIND_SERVICE_CONTENT)
+    processutils.execute('systemctl', 'enable', 'sriov_bind')
+
+
+def bind_vfs(sriov_bind_pcis_map=None):
+    if not sriov_bind_pcis_map:
+        sriov_bind_pcis_map = _get_sriov_bind_pcis_map()
+    for driver, pcis_list in sriov_bind_pcis_map.items():
+        for vf_pci in pcis_list:
+            vf_pci_driver_path = _PCI_DRIVER_FILE_PATH % {"pci": vf_pci}
+            if not os.path.exists(vf_pci_driver_path):
+                pci_driver_bind_file_path = _PCI_DRIVER_BIND_FILE_PATH %\
+                    {"driver": driver}
+                try:
+                    with open(pci_driver_bind_file_path, 'w') as f:
+                        f.write("%s" % vf_pci)
+                    logger.info("Vf %s has been bound" % vf_pci)
+                except IOError:
+                    logger.error("Failed to bind vf %s" % vf_pci)
+
+
+def main():
+    bind_vfs()
+
+
+if __name__ == "__main__":
+    main()

os_net_config/sriov_config.py

@@ -31,6 +31,7 @@ import sys
 import time
 import yaml
 
+from os_net_config import sriov_bind_config
 from oslo_concurrency import processutils
 
 logger = logging.getLogger(__name__)
@@ -40,6 +41,8 @@ _UDEV_LEGACY_RULE_FILE = '/etc/udev/rules.d/70-os-net-config-sriov.rules'
 _IFUP_LOCAL_FILE = '/sbin/ifup-local'
 _RESET_SRIOV_RULES_FILE = '/etc/udev/rules.d/70-tripleo-reset-sriov.rules'
 _ALLOCATE_VFS_FILE = '/etc/sysconfig/allocate_vfs'
+_MLNX_DRIVER = "mlx5_core"
+MLNX_VENDOR_ID = "0x15b3"
 
 MAX_RETRIES = 10
 PF_FUNC_RE = re.compile(r"\.(\d+)$", 0)
@@ -171,7 +174,7 @@ def configure_sriov_pf(execution_from_cli=False, restart_openvswitch=False):
 
     sriov_map = _get_sriov_map()
     MLNX_UNBIND_FILE_PATH = "/sys/bus/pci/drivers/mlx5_core/unbind"
-    MLNX_VENDOR_ID = "0x15b3"
+    mlnx_vfs_pcis_list = []
     trigger_udev_rule = False
 
     # Cleanup the previous config by puppet-tripleo
@@ -206,6 +209,7 @@ def configure_sriov_pf(execution_from_cli=False, restart_openvswitch=False):
             if (item.get('link_mode') == "switchdev" and
                     vendor_id == MLNX_VENDOR_ID):
                 vf_pcis_list = get_vf_pcis_list(item['name'])
+                mlnx_vfs_pcis_list += vf_pcis_list
                 for vf_pci in vf_pcis_list:
                     vf_pci_path = "/sys/bus/pci/devices/%s/driver" % vf_pci
                     if os.path.exists(vf_pci_path):
@@ -238,6 +242,14 @@ def configure_sriov_pf(execution_from_cli=False, restart_openvswitch=False):
                 if execution_from_cli:
                     if_up_interface(item['name'])
 
+    if mlnx_vfs_pcis_list:
+        sriov_bind_pcis_map = {_MLNX_DRIVER: mlnx_vfs_pcis_list}
+        if not execution_from_cli:
+            sriov_bind_config.update_sriov_bind_pcis_map(sriov_bind_pcis_map)
+        else:
+            sriov_bind_config.configure_sriov_bind_service()
+            sriov_bind_config.bind_vfs(sriov_bind_pcis_map)
+
     # Trigger udev rules if there is new rules written
     if trigger_udev_rule:
         trigger_udev_rules()

os_net_config/tests/test_sriov_bind_config.py

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2019 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+import os.path
+import random
+
+
+from os_net_config import sriov_bind_config
+from os_net_config.tests import base
+from os_net_config import utils
+
+
+class TestSriovBindConfig(base.TestCase):
+    """Unit tests for methods defined in sriov_bind_config.py"""
+
+    def setUp(self):
+        super(TestSriovBindConfig, self).setUp()
+        rand = str(int(random.random() * 100000))
+
+        sriov_bind_config._SRIOV_BIND_CONFIG_FILE = '/tmp/' + rand +\
+            'sriov_bind_config.yaml'
+        sriov_bind_config._PCI_DRIVER_BIND_FILE_PATH = '/tmp/' + rand +\
+            '%(driver)s/bind'
+
+    def tearDown(self):
+        super(TestSriovBindConfig, self).tearDown()
+        if os.path.isfile(sriov_bind_config._SRIOV_BIND_CONFIG_FILE):
+            os.remove(sriov_bind_config._SRIOV_BIND_CONFIG_FILE)
+
+    def test_bind_vfs(self):
+        """Test SR-IOV VFs binding"""
+        vfs_driver = "mlx5_core"
+        sriov_bind_pcis_map = {vfs_driver: ['0000:03:00.2', '0000:03:00.3']}
+        os.makedirs(sriov_bind_config._PCI_DRIVER_BIND_FILE_PATH %
+                    {"driver": vfs_driver})
+
+        utils.write_yaml_config(sriov_bind_config._SRIOV_BIND_CONFIG_FILE,
+                                sriov_bind_pcis_map)
+        sriov_bind_config.bind_vfs()

setup.cfg

@@ -30,3 +30,4 @@ packages =
 console_scripts =
         os-net-config = os_net_config.cli:main
         os-net-config-sriov = os_net_config.sriov_config:main
+        os-net-config-sriov-bind = os_net_config.sriov_bind_config:main