Support DPDK application on bare-metal host

This patch contains binding driver, which intend to copy vhostuser port
to containers's directory. Here container's directory it's mounted
directory. Also this patch contains code to create proper VIF in case
when neutron ovs agent configured to work with vhostuser ports.

There is no code here for port creation, due to it performs in
base.connect by os_vif.plug. This function creates/or recreates OVS
bridge with netdev type, then it  creates port in this bridge. It uses
vif.network.bridge as the name for integration bridge, IOW it doesn't
use ovs_bridge from kuryr.conf, vif.network.bridge is configured by
neutron ovs agent.

VhostUser mode is defined by neutron ovs agent, it obtains from
Open vSwitch configuration:

Command to check Open vSwitch configuration

ovs-vsctl list Open_vSwitch |grep iface_types

If neutron ovs agent finds dpdkvhostuserclient there, it sets
vhostuser_mode to VIFVHostUserMode.SERVER, it means DPDK application in
container will be a server, and OVS will be a client, so DPDK
application will create/bind/listen vhostuser socket by predefined path.
This path is set in kuryr.conf/vhostuser/mount_point.
When dpdkvhostuserclient is not in OVS's capability list, e.g. it's old
OVS or it was built w/o dpdkvhostuserclient support, the mode will be
VIFVHostUserMode.CLIENT. In this case OVS will create/bind/listen
socket, so socket file will exist, and it shoud be copied to container's
mount volume. At the moment of copying OVS server already has to listen
it, otherwise approach is not working.

Partially Implements: blueprint support-vhost-user-port-type-on-bm-installation

Change-Id: Ib9c22368e518815064282f4c3b9f9ddaf58dc622
Signed-off-by: Alexey Perevalov <a.perevalov@samsung.com>
Signed-off-by: Andrey Zaikin <a.zaikin@partner.samsung.com>
Signed-off-by: Vladimir Kuramshin <v.kuramshin@samsung.com>

kuryr_kubernetes/cni/binding/vhostuser.py

@@ -0,0 +1,138 @@
+# Copyright (c) 2020 Samsung Electronics Co., Ltd.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+import os.path
+import stat
+
+from kuryr.lib._i18n import _
+from os_vif.objects import fields as osv_fields
+from oslo_config import cfg
+from oslo_log import log
+from oslo_serialization import jsonutils
+from vif_plug_ovs import constants
+from vif_plug_ovs import ovs
+
+from kuryr_kubernetes.cni.binding import base
+from kuryr_kubernetes import config
+from kuryr_kubernetes import exceptions as k_exc
+from kuryr_kubernetes.handlers import health
+
+LOG = log.getLogger(__name__)
+
+
+def _get_vhostport_type(vif):
+    if vif.mode == osv_fields.VIFVHostUserMode.SERVER:
+        return 'dpdkvhostuserclient'
+    elif vif.mode == osv_fields.VIFVHostUserMode.CLIENT:
+        return 'dpdkvhostuser'
+    raise k_exc.IntegrityError(
+        _("Unknown vhostuser mode %(mode)s for vif %(vif_id)s")
+        % {'mode': vif.mode, 'vif_id': vif.id})
+
+
+def _get_vhostuser_port_name(vif):
+    return ovs.OvsPlugin.gen_port_name(constants.OVS_VHOSTUSER_PREFIX, vif.id)
+
+
+def _get_vhu_sock(config_file_path):
+    with open(config_file_path, 'r') as f:
+        conf = jsonutils.load(f)
+    return conf['vhostname']
+
+
+def _check_sock_file(vhostuser_socket):
+    mode = os.stat(vhostuser_socket).st_mode
+    return stat.S_ISSOCK(mode)
+
+
+class VIFVHostUserDriver(health.HealthHandler, base.BaseBindingDriver):
+
+    def __init__(self):
+        super(VIFVHostUserDriver, self).__init__()
+        self.mount_path = config.CONF.vhostuser.mount_point
+        self.ovs_vu_path = config.CONF.vhostuser.ovs_vhu_path
+        if not self.mount_path:
+            raise cfg.RequiredOptError('mount_point', 'vhostuser')
+
+    def _write_config(self, container_id, ifname, port_name, vif):
+        """Write vhostuser configuration file
+
+        This function writes configuration file, this file will be used by
+        application inside container and for cleanup (def disconnect)
+        procedure.
+        """
+        vhost_conf = {}
+        vhost_conf["vhostname"] = port_name
+        vhost_conf["vhostmac"] = vif.address
+        vhost_conf["mode"] = vif.mode
+        with open(self._config_file_path(container_id, ifname), "w") as f:
+            jsonutils.dump(vhost_conf, f)
+
+    def _config_file_path(self, container_id, ifname):
+        return os.path.join(self.mount_path, f'{container_id}-{ifname}')
+
+    def connect(self, vif, ifname, netns, container_id):
+        port_name = _get_vhostuser_port_name(vif)
+        self._write_config(container_id, ifname, port_name, vif)
+        # no need to copy in case of SERVER mode
+        if vif.mode == osv_fields.VIFVHostUserMode.SERVER:
+            return
+
+        src_vhu_sock = os.path.join(self.ovs_vu_path, port_name)
+
+        if _check_sock_file(src_vhu_sock):
+            dst_vhu_sock = os.path.join(vif.path, port_name)
+            LOG.debug("Moving %s to %s while processing VIF %s", src_vhu_sock,
+                      dst_vhu_sock, vif.id)
+            os.rename(src_vhu_sock, dst_vhu_sock)
+        else:
+            error_msg = "Socket %s required for VIF %s doesn't exist".format(
+                src_vhu_sock, vif.id)
+            LOG.error(error_msg)
+            raise k_exc.CNIError(error_msg)
+
+    def disconnect(self, vif, ifname, netns, container_id):
+        # This function removes configuration file and appropriate
+        # socket file. Unfortunatelly Open vSwitch daemon can't remove
+        # moved socket, so we have to do it
+        config_file_path = self._config_file_path(container_id, ifname)
+
+        if not os.path.exists(config_file_path):
+            LOG.warning("Configuration file: %s for VIF %s doesn't exist!",
+                        config_file_path, vif.id)
+            return
+        vhu_sock_path = os.path.join(self.mount_path,
+                                     _get_vhu_sock(config_file_path))
+        LOG.debug("remove: %s, %s", config_file_path, vhu_sock_path)
+        os.remove(vhu_sock_path)
+        os.remove(config_file_path)
+
+    def is_alive(self):
+        healthy = False
+        try:
+            healthy = (os.path.exists(self.ovs_vu_path)
+                       and os.path.exists(self.mount_path))
+        except Exception:
+            LOG.exception('Error when determining health status of vhostuser '
+                          'CNI driver.')
+
+        if not healthy:
+            LOG.error('Directory %s or %s does not exist or Kuryr has no '
+                      'permissions to access it. Marking vhostuser binding '
+                      'driver as unhealthy.', self.ovs_vu_path,
+                      self.mount_path)
+
+        return healthy

kuryr_kubernetes/config.py

@@ -313,6 +313,17 @@ sriov_opts = [
 ]
 
 
+vhostuser = [
+    cfg.StrOpt('mount_point',
+               help=_("Path where vhost-user port will be created "
+                      "also it should be mount point for pod"),
+               default='/var/cni/vhostuser'),
+    cfg.StrOpt('ovs_vhu_path',
+               help=_("Path where OVS keeps socket files for vhost-user "
+                      "ports"),
+               default='/var/run/openvswitch/')
+]
+
 CONF = cfg.CONF
 CONF.register_opts(kuryr_k8s_opts)
 CONF.register_opts(daemon_opts, group='cni_daemon')
@@ -322,6 +333,7 @@ CONF.register_opts(octavia_defaults, group='octavia_defaults')
 CONF.register_opts(cache_defaults, group='cache_defaults')
 CONF.register_opts(nested_vif_driver_opts, group='pod_vif_nested')
 CONF.register_opts(sriov_opts, group='sriov')
+CONF.register_opts(vhostuser, group='vhostuser')
 
 CONF.register_opts(lib_config.core_opts)
 CONF.register_opts(lib_config.binding_opts, 'binding')

kuryr_kubernetes/controller/drivers/vif_pool.py

@@ -100,6 +100,7 @@ VIF_TYPE_TO_DRIVER_MAPPING = {
     'VIFMacvlanNested': 'nested-macvlan',
     'VIFSriov': 'sriov',
     'VIFDPDKNested': 'nested-dpdk',
+    'VIFVHostUser': 'neutron-vif',
 }
 
 

kuryr_kubernetes/os_vif_util.py

@@ -23,7 +23,9 @@ from os_vif.objects import route as osv_route
 from os_vif.objects import subnet as osv_subnet
 from os_vif.objects import vif as osv_vif
 from oslo_config import cfg as oslo_cfg
+from oslo_log import log as logging
 from stevedore import driver as stv_driver
+from vif_plug_ovs import constants as osv_const
 
 from kuryr_kubernetes import config
 from kuryr_kubernetes import constants as const
@@ -31,6 +33,8 @@ from kuryr_kubernetes import exceptions as k_exc
 from kuryr_kubernetes.objects import vif as k_vif
 
 
+LOG = logging.getLogger(__name__)
+
 # REVISIT(ivc): consider making this module part of kuryr-lib
 _VIF_TRANSLATOR_NAMESPACE = "kuryr_kubernetes.vif_translators"
 _VIF_MANAGERS = {}
@@ -193,6 +197,15 @@ def _make_vif_network(neutron_port, subnets):
     return network
 
 
+# TODO(a.perevalov) generalize it with get_veth_pair_names
+# but it's reasonable if we're going to add vhostuser support
+# into kuryr project
+def _get_vhu_vif_name(port_id):
+    ifname = osv_const.OVS_VHOSTUSER_PREFIX + port_id
+    ifname = ifname[:kl_const.NIC_NAME_LEN]
+    return ifname
+
+
 def _get_vif_name(neutron_port):
     """Gets a VIF device name for port.
 
@@ -249,8 +262,30 @@ def neutron_to_osvif_vif_ovs(vif_plugin, os_port, subnets):
 
     network = _make_vif_network(os_port, subnets)
     network.bridge = ovs_bridge
+    vhostuser_mode = details.get('vhostuser_mode', False)
 
-    if details.get('ovs_hybrid_plug'):
+    LOG.debug('Detected vhostuser_mode=%s for port %s', vhostuser_mode,
+              os_port.id)
+    if vhostuser_mode:
+        # TODO(a.perevalov) obtain path to mount point from pod's mountVolumes
+        vhostuser_mount_point = (config.CONF.vhostuser.mount_point)
+        if not vhostuser_mount_point:
+            raise oslo_cfg.RequiredOptError('vhostuser_mount_point',
+                                            'neutron_defaults')
+        vif = osv_vif.VIFVHostUser(
+            id=os_port.id,
+            address=os_port.mac_address,
+            network=network,
+            has_traffic_filtering=details.get('port_filter', False),
+            preserve_on_delete=False,
+            active=_is_port_active(os_port),
+            port_profile=profile,
+            plugin='ovs',
+            path=vhostuser_mount_point,
+            mode=vhostuser_mode,
+            vif_name=_get_vhu_vif_name(os_port.id),
+            bridge_name=network.bridge)
+    elif details.get('ovs_hybrid_plug'):
         vif = osv_vif.VIFBridge(
             id=os_port.id,
             address=os_port.mac_address,
@@ -386,7 +421,6 @@ def neutron_to_osvif_vif(vif_translator, os_port, subnets):
     :param subnets: subnet mapping as returned by PodSubnetsDriver.get_subnets
     :return: os-vif VIF object
     """
-
     try:
         mgr = _VIF_MANAGERS[vif_translator]
     except KeyError:

kuryr_kubernetes/tests/unit/cni/test_binding.py

@@ -13,13 +13,18 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 import mock
+import os
 import uuid
 
+
 from os_vif import objects as osv_objects
+from os_vif.objects import fields as osv_fields
 from oslo_config import cfg
+from oslo_utils import uuidutils
 
 from kuryr_kubernetes.cni.binding import base
 from kuryr_kubernetes.cni.binding import sriov
+from kuryr_kubernetes.cni.binding import vhostuser
 from kuryr_kubernetes import constants as k_const
 from kuryr_kubernetes import exceptions
 from kuryr_kubernetes import objects
@@ -402,3 +407,92 @@ class TestSriovDriver(TestDriverMixin, test_base.TestCase):
         cls._return_device_driver(m_driver, self.vif)
         m_driver._bind_device.assert_called_once_with(pci, old_driver,
                                                       new_driver)
+
+
+class TestVHostUserDriver(TestDriverMixin, test_base.TestCase):
+    def setUp(self):
+        super(TestVHostUserDriver, self).setUp()
+        self.vu_mount_point = '/var/run/cni'
+        self.vu_ovs_path = '/var/run/openvswitch'
+        CONF.set_override('mount_point', self.vu_mount_point,
+                          group='vhostuser')
+        CONF.set_override('ovs_vhu_path', self.vu_ovs_path,
+                          group='vhostuser')
+        self.vif = fake._fake_vif(osv_objects.vif.VIFVHostUser)
+        self.vif.path = self.vu_mount_point
+        self.vif.address = '64:0f:2b:5f:0c:1c'
+        self.port_name = vhostuser._get_vhostuser_port_name(self.vif)
+        self.cont_id = uuidutils.generate_uuid()
+
+    @mock.patch('kuryr_kubernetes.cni.binding.base._need_configure_l3')
+    @mock.patch('kuryr_kubernetes.cni.plugins.k8s_cni_registry.'
+                'K8sCNIRegistryPlugin.report_drivers_health')
+    @mock.patch('os.rename')
+    @mock.patch('os.path.exists', mock.Mock(return_value=True))
+    @mock.patch('kuryr_kubernetes.cni.binding.vhostuser.VIFVHostUserDriver.'
+                '_write_config')
+    @mock.patch('kuryr_kubernetes.cni.binding.vhostuser._check_sock_file')
+    @mock.patch('os_vif.plug')
+    def test_connect_client(self, m_vif_plug, m_check_sock, m_write_conf,
+                            m_os_rename, m_report, m_need_l3):
+        m_need_l3.return_value = False
+        self.vif.mode = osv_fields.VIFVHostUserMode.CLIENT
+        m_check_sock.return_value = True
+        base.connect(self.vif, self.instance_info, self.ifname, self.netns,
+                     m_report, container_id=self.cont_id)
+        vu_dst_socket = os.path.join(self.vu_mount_point, self.port_name)
+        vu_src_socket = os.path.join(self.vu_ovs_path, self.port_name)
+
+        m_vif_plug.assert_called_once_with(self.vif, self.instance_info)
+        m_os_rename.assert_called_once_with(vu_src_socket, vu_dst_socket)
+        m_write_conf.assert_called_once_with(self.cont_id, self.ifname,
+                                             self.port_name, self.vif)
+        m_report.assert_called_once()
+
+    @mock.patch('kuryr_kubernetes.cni.binding.base._need_configure_l3')
+    @mock.patch('kuryr_kubernetes.cni.plugins.k8s_cni_registry.'
+                'K8sCNIRegistryPlugin.report_drivers_health')
+    @mock.patch('kuryr_kubernetes.cni.binding.vhostuser.VIFVHostUserDriver.'
+                '_write_config')
+    @mock.patch('os_vif.plug')
+    def test_connect_server(self, m_vif_plug, m_write_conf,
+                            m_report, m_need_l3):
+        m_need_l3.return_value = False
+        self.vif.mode = osv_fields.VIFVHostUserMode.SERVER
+        base.connect(self.vif, self.instance_info, self.ifname, self.netns,
+                     m_report, container_id=self.cont_id)
+        m_vif_plug.assert_called_once_with(self.vif, self.instance_info)
+        m_write_conf.assert_called_once_with(self.cont_id, self.ifname,
+                                             self.port_name, self.vif)
+        m_report.assert_called_once()
+
+    @mock.patch('kuryr_kubernetes.cni.plugins.k8s_cni_registry.'
+                'K8sCNIRegistryPlugin.report_drivers_health')
+    @mock.patch('kuryr_kubernetes.cni.binding.vhostuser._check_sock_file',
+                mock.Mock(return_value=False))
+    @mock.patch('kuryr_kubernetes.cni.binding.vhostuser.VIFVHostUserDriver.'
+                '_write_config', mock.Mock())
+    @mock.patch('os_vif.plug')
+    def test_connect_nosocket(self, m_vif_plug, m_report):
+        self.vif.mode = osv_fields.VIFVHostUserMode.CLIENT
+        self.assertRaises(exceptions.CNIError, base.connect, self.vif,
+                          self.instance_info, self.ifname, self.netns,
+                          m_report, container_id=self.cont_id)
+
+    @mock.patch('kuryr_kubernetes.cni.plugins.k8s_cni_registry.'
+                'K8sCNIRegistryPlugin.report_drivers_health')
+    @mock.patch('kuryr_kubernetes.cni.binding.vhostuser._get_vhu_sock')
+    @mock.patch('os.remove')
+    @mock.patch('os.path.exists', mock.Mock(return_value=True))
+    @mock.patch('os_vif.unplug')
+    def test_disconnect(self, m_os_unplug, m_os_remove, m_get_vhu_sock,
+                        m_report):
+        m_get_vhu_sock.return_value = self.port_name
+        base.disconnect(self.vif, self.instance_info, self.ifname, self.netns,
+                        m_report, container_id=self.cont_id)
+        conf_file_path = '{}/{}-{}'.format(self.vu_mount_point,
+                                           self.cont_id, self.ifname)
+        vhu_sock_path = '{}/{}'.format(self.vu_mount_point,
+                                       self.port_name)
+        os_remove_calls = [mock.call(vhu_sock_path), mock.call(conf_file_path)]
+        m_os_remove.assert_has_calls(os_remove_calls)

kuryr_kubernetes/tests/unit/test_os_vif_util.py

@@ -184,7 +184,7 @@ class TestOSVIFUtils(test_base.TestCase):
         port_profile = mock.sentinel.port_profile
         network = mock.sentinel.network
         port_active = mock.sentinel.port_active
-        vif_name = mock.sentinel.vif_name
+        vif_name = "vhu01234567-89"
         hybrid_bridge = mock.sentinel.hybrid_bridge
         vif = mock.sentinel.vif
         port = fake.get_port_obj(port_id=port_id,
@@ -240,7 +240,7 @@ class TestOSVIFUtils(test_base.TestCase):
 
         subnets = mock.sentinel.subnets
         network = mock.sentinel.network
-        vif_name = mock.sentinel.vif_name
+        vif_name = "vhu01234567-89"
         vif = mock.sentinel.vif
 
         m_mk_profile.return_value = port.profile
@@ -258,6 +258,92 @@ class TestOSVIFUtils(test_base.TestCase):
         self.assertEqual(network.bridge,
                          port.binding_vif_details['bridge_name'])
 
+    @mock.patch('kuryr_kubernetes.os_vif_util._get_vhu_vif_name')
+    @mock.patch('kuryr_kubernetes.os_vif_util._is_port_active')
+    @mock.patch('kuryr_kubernetes.os_vif_util._make_vif_network')
+    @mock.patch('os_vif.objects.vif.VIFVHostUser')
+    @mock.patch('os_vif.objects.vif.VIFPortProfileOpenVSwitch')
+    def test_neutron_to_osvif_vif_ovs_vu_client(self, m_mk_profile, m_mk_vif,
+                                                m_make_vif_network,
+                                                m_is_port_active,
+                                                m_get_vif_name):
+        vif_plugin = 'vhostuser'
+        o_cfg.CONF.set_override('mount_point',
+                                '/var/lib/cni/vhostuser',
+                                group='vhostuser')
+        port_id = mock.sentinel.port_id
+        mac_address = mock.sentinel.mac_address
+        ovs_bridge = mock.sentinel.ovs_bridge
+        subnets = mock.sentinel.subnets
+        port_profile = mock.sentinel.port_profile
+        network = mock.sentinel.network
+        port_active = mock.sentinel.port_active
+        vif_name = "vhu01234567-89"
+        vif = mock.sentinel.vif
+
+        m_mk_profile.return_value = port_profile
+        m_make_vif_network.return_value = network
+        m_is_port_active.return_value = port_active
+        m_get_vif_name.return_value = vif_name
+        m_mk_vif.return_value = vif
+
+        port = fake.get_port_obj(port_id=port_id,
+                                 vif_details={'ovs_hybrid_plug': False,
+                                              'bridge_name': ovs_bridge,
+                                              'vhostuser_mode': 'client'})
+        port.mac_address = mac_address
+
+        self.assertEqual(vif, ovu.neutron_to_osvif_vif_ovs(vif_plugin, port,
+                                                           subnets))
+        m_mk_profile.assert_called_once_with(interface_id=port_id)
+        m_make_vif_network.assert_called_once_with(port, subnets)
+        m_is_port_active.assert_called_once_with(port)
+        m_get_vif_name.assert_called_once_with(port_id)
+        self.assertEqual(ovs_bridge, network.bridge)
+
+    @mock.patch('kuryr_kubernetes.os_vif_util._get_vhu_vif_name')
+    @mock.patch('kuryr_kubernetes.os_vif_util._is_port_active')
+    @mock.patch('kuryr_kubernetes.os_vif_util._make_vif_network')
+    @mock.patch('os_vif.objects.vif.VIFVHostUser')
+    @mock.patch('os_vif.objects.vif.VIFPortProfileOpenVSwitch')
+    def test_neutron_to_osvif_vif_ovs_vu_server(self, m_mk_profile, m_mk_vif,
+                                                m_make_vif_network,
+                                                m_is_port_active,
+                                                m_get_vif_name):
+        vif_plugin = 'vhostuser'
+        o_cfg.CONF.set_override('mount_point',
+                                '/var/lib/cni/vhostuser',
+                                group='vhostuser')
+        port_id = mock.sentinel.port_id
+        mac_address = mock.sentinel.mac_address
+        ovs_bridge = mock.sentinel.ovs_bridge
+        subnets = mock.sentinel.subnets
+        port_profile = mock.sentinel.port_profile
+        network = mock.sentinel.network
+        port_active = mock.sentinel.port_active
+        vif_name = mock.sentinel.vif_name
+        vif = mock.sentinel.vif
+
+        m_mk_profile.return_value = port_profile
+        m_make_vif_network.return_value = network
+        m_is_port_active.return_value = port_active
+        m_get_vif_name.return_value = vif_name
+        m_mk_vif.return_value = vif
+
+        port = fake.get_port_obj(port_id=port_id,
+                                 vif_details={'ovs_hybrid_plug': False,
+                                              'bridge_name': ovs_bridge,
+                                              'vhostuser_mode': 'server'})
+        port.mac_address = mac_address
+
+        self.assertEqual(vif, ovu.neutron_to_osvif_vif_ovs(vif_plugin, port,
+                                                           subnets))
+        m_mk_profile.assert_called_once_with(interface_id=port_id)
+        m_make_vif_network.assert_called_once_with(port, subnets)
+        m_is_port_active.assert_called_once_with(port)
+        m_get_vif_name.assert_called_once_with(port_id)
+        self.assertEqual(ovs_bridge, network.bridge)
+
     @mock.patch('kuryr_kubernetes.os_vif_util._get_vif_name')
     @mock.patch('kuryr_kubernetes.os_vif_util._is_port_active')
     @mock.patch('kuryr_kubernetes.os_vif_util._make_vif_network')

setup.cfg

@@ -35,11 +35,13 @@ console_scripts =
 kuryr_kubernetes.vif_translators =
     ovs = kuryr_kubernetes.os_vif_util:neutron_to_osvif_vif_ovs
     sriov = kuryr_kubernetes.os_vif_util:neutron_to_osvif_vif_sriov
+    vhostuser = kuryr_kubernetes.os_vif_util:neutron_to_osvif_vif_ovs
 
 kuryr_kubernetes.cni.binding =
     VIFBridge = kuryr_kubernetes.cni.binding.bridge:BridgeDriver
     VIFOpenVSwitch = kuryr_kubernetes.cni.binding.bridge:VIFOpenVSwitchDriver
     VIFDPDKNested = kuryr_kubernetes.cni.binding.dpdk:DpdkDriver
+    VIFVHostUser = kuryr_kubernetes.cni.binding.vhostuser:VIFVHostUserDriver
     VIFVlanNested = kuryr_kubernetes.cni.binding.nested:VlanDriver
     VIFMacvlanNested = kuryr_kubernetes.cni.binding.nested:MacvlanDriver
     VIFSriov = kuryr_kubernetes.cni.binding.sriov:VIFSriovDriver