Merge "Support DPDK application on bare-metal host"

kuryr_kubernetes/cni/binding/vhostuser.py

@@ -0,0 +1,138 @@
+# Copyright (c) 2020 Samsung Electronics Co., Ltd.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+import os.path
+import stat
+
+from kuryr.lib._i18n import _
+from os_vif.objects import fields as osv_fields
+from oslo_config import cfg
+from oslo_log import log
+from oslo_serialization import jsonutils
+from vif_plug_ovs import constants
+from vif_plug_ovs import ovs
+
+from kuryr_kubernetes.cni.binding import base
+from kuryr_kubernetes import config
+from kuryr_kubernetes import exceptions as k_exc
+from kuryr_kubernetes.handlers import health
+
+LOG = log.getLogger(__name__)
+
+
+def _get_vhostport_type(vif):
+    if vif.mode == osv_fields.VIFVHostUserMode.SERVER:
+        return 'dpdkvhostuserclient'
+    elif vif.mode == osv_fields.VIFVHostUserMode.CLIENT:
+        return 'dpdkvhostuser'
+    raise k_exc.IntegrityError(
+        _("Unknown vhostuser mode %(mode)s for vif %(vif_id)s")
+        % {'mode': vif.mode, 'vif_id': vif.id})
+
+
+def _get_vhostuser_port_name(vif):
+    return ovs.OvsPlugin.gen_port_name(constants.OVS_VHOSTUSER_PREFIX, vif.id)
+
+
+def _get_vhu_sock(config_file_path):
+    with open(config_file_path, 'r') as f:
+        conf = jsonutils.load(f)
+    return conf['vhostname']
+
+
+def _check_sock_file(vhostuser_socket):
+    mode = os.stat(vhostuser_socket).st_mode
+    return stat.S_ISSOCK(mode)
+
+
+class VIFVHostUserDriver(health.HealthHandler, base.BaseBindingDriver):
+
+    def __init__(self):
+        super(VIFVHostUserDriver, self).__init__()
+        self.mount_path = config.CONF.vhostuser.mount_point
+        self.ovs_vu_path = config.CONF.vhostuser.ovs_vhu_path
+        if not self.mount_path:
+            raise cfg.RequiredOptError('mount_point', 'vhostuser')
+
+    def _write_config(self, container_id, ifname, port_name, vif):
+        """Write vhostuser configuration file
+
+        This function writes configuration file, this file will be used by
+        application inside container and for cleanup (def disconnect)
+        procedure.
+        """
+        vhost_conf = {}
+        vhost_conf["vhostname"] = port_name
+        vhost_conf["vhostmac"] = vif.address
+        vhost_conf["mode"] = vif.mode
+        with open(self._config_file_path(container_id, ifname), "w") as f:
+            jsonutils.dump(vhost_conf, f)
+
+    def _config_file_path(self, container_id, ifname):
+        return os.path.join(self.mount_path, f'{container_id}-{ifname}')
+
+    def connect(self, vif, ifname, netns, container_id):
+        port_name = _get_vhostuser_port_name(vif)
+        self._write_config(container_id, ifname, port_name, vif)
+        # no need to copy in case of SERVER mode
+        if vif.mode == osv_fields.VIFVHostUserMode.SERVER:
+            return
+
+        src_vhu_sock = os.path.join(self.ovs_vu_path, port_name)
+
+        if _check_sock_file(src_vhu_sock):
+            dst_vhu_sock = os.path.join(vif.path, port_name)
+            LOG.debug("Moving %s to %s while processing VIF %s", src_vhu_sock,
+                      dst_vhu_sock, vif.id)
+            os.rename(src_vhu_sock, dst_vhu_sock)
+        else:
+            error_msg = "Socket %s required for VIF %s doesn't exist".format(
+                src_vhu_sock, vif.id)
+            LOG.error(error_msg)
+            raise k_exc.CNIError(error_msg)
+
+    def disconnect(self, vif, ifname, netns, container_id):
+        # This function removes configuration file and appropriate
+        # socket file. Unfortunatelly Open vSwitch daemon can't remove
+        # moved socket, so we have to do it
+        config_file_path = self._config_file_path(container_id, ifname)
+
+        if not os.path.exists(config_file_path):
+            LOG.warning("Configuration file: %s for VIF %s doesn't exist!",
+                        config_file_path, vif.id)
+            return
+        vhu_sock_path = os.path.join(self.mount_path,
+                                     _get_vhu_sock(config_file_path))
+        LOG.debug("remove: %s, %s", config_file_path, vhu_sock_path)
+        os.remove(vhu_sock_path)
+        os.remove(config_file_path)
+
+    def is_alive(self):
+        healthy = False
+        try:
+            healthy = (os.path.exists(self.ovs_vu_path)
+                       and os.path.exists(self.mount_path))
+        except Exception:
+            LOG.exception('Error when determining health status of vhostuser '
+                          'CNI driver.')
+
+        if not healthy:
+            LOG.error('Directory %s or %s does not exist or Kuryr has no '
+                      'permissions to access it. Marking vhostuser binding '
+                      'driver as unhealthy.', self.ovs_vu_path,
+                      self.mount_path)
+
+        return healthy

kuryr_kubernetes/config.py

@@ -319,6 +319,17 @@ sriov_opts = [
 ]
 
 
+vhostuser = [
+    cfg.StrOpt('mount_point',
+               help=_("Path where vhost-user port will be created "
+                      "also it should be mount point for pod"),
+               default='/var/cni/vhostuser'),
+    cfg.StrOpt('ovs_vhu_path',
+               help=_("Path where OVS keeps socket files for vhost-user "
+                      "ports"),
+               default='/var/run/openvswitch/')
+]
+
 CONF = cfg.CONF
 CONF.register_opts(kuryr_k8s_opts)
 CONF.register_opts(daemon_opts, group='cni_daemon')
@@ -328,6 +339,7 @@ CONF.register_opts(octavia_defaults, group='octavia_defaults')
 CONF.register_opts(cache_defaults, group='cache_defaults')
 CONF.register_opts(nested_vif_driver_opts, group='pod_vif_nested')
 CONF.register_opts(sriov_opts, group='sriov')
+CONF.register_opts(vhostuser, group='vhostuser')
 
 CONF.register_opts(lib_config.core_opts)
 CONF.register_opts(lib_config.binding_opts, 'binding')

kuryr_kubernetes/controller/drivers/vif_pool.py

@@ -100,6 +100,7 @@ VIF_TYPE_TO_DRIVER_MAPPING = {
     'VIFMacvlanNested': 'nested-macvlan',
     'VIFSriov': 'sriov',
     'VIFDPDKNested': 'nested-dpdk',
+    'VIFVHostUser': 'neutron-vif',
 }
 
 

kuryr_kubernetes/os_vif_util.py

@@ -23,7 +23,9 @@ from os_vif.objects import route as osv_route
 from os_vif.objects import subnet as osv_subnet
 from os_vif.objects import vif as osv_vif
 from oslo_config import cfg as oslo_cfg
+from oslo_log import log as logging
 from stevedore import driver as stv_driver
+from vif_plug_ovs import constants as osv_const
 
 from kuryr_kubernetes import config
 from kuryr_kubernetes import constants as const
@@ -31,6 +33,8 @@ from kuryr_kubernetes import exceptions as k_exc
 from kuryr_kubernetes.objects import vif as k_vif
 
 
+LOG = logging.getLogger(__name__)
+
 # REVISIT(ivc): consider making this module part of kuryr-lib
 _VIF_TRANSLATOR_NAMESPACE = "kuryr_kubernetes.vif_translators"
 _VIF_MANAGERS = {}
@@ -193,6 +197,15 @@ def _make_vif_network(neutron_port, subnets):
     return network
 
 
+# TODO(a.perevalov) generalize it with get_veth_pair_names
+# but it's reasonable if we're going to add vhostuser support
+# into kuryr project
+def _get_vhu_vif_name(port_id):
+    ifname = osv_const.OVS_VHOSTUSER_PREFIX + port_id
+    ifname = ifname[:kl_const.NIC_NAME_LEN]
+    return ifname
+
+
 def _get_vif_name(neutron_port):
     """Gets a VIF device name for port.
 
@@ -249,8 +262,30 @@ def neutron_to_osvif_vif_ovs(vif_plugin, os_port, subnets):
 
     network = _make_vif_network(os_port, subnets)
     network.bridge = ovs_bridge
+    vhostuser_mode = details.get('vhostuser_mode', False)
 
-    if details.get('ovs_hybrid_plug'):
+    LOG.debug('Detected vhostuser_mode=%s for port %s', vhostuser_mode,
+              os_port.id)
+    if vhostuser_mode:
+        # TODO(a.perevalov) obtain path to mount point from pod's mountVolumes
+        vhostuser_mount_point = (config.CONF.vhostuser.mount_point)
+        if not vhostuser_mount_point:
+            raise oslo_cfg.RequiredOptError('vhostuser_mount_point',
+                                            'neutron_defaults')
+        vif = osv_vif.VIFVHostUser(
+            id=os_port.id,
+            address=os_port.mac_address,
+            network=network,
+            has_traffic_filtering=details.get('port_filter', False),
+            preserve_on_delete=False,
+            active=_is_port_active(os_port),
+            port_profile=profile,
+            plugin='ovs',
+            path=vhostuser_mount_point,
+            mode=vhostuser_mode,
+            vif_name=_get_vhu_vif_name(os_port.id),
+            bridge_name=network.bridge)
+    elif details.get('ovs_hybrid_plug'):
         vif = osv_vif.VIFBridge(
             id=os_port.id,
             address=os_port.mac_address,
@@ -386,7 +421,6 @@ def neutron_to_osvif_vif(vif_translator, os_port, subnets):
     :param subnets: subnet mapping as returned by PodSubnetsDriver.get_subnets
     :return: os-vif VIF object
     """
-
     try:
         mgr = _VIF_MANAGERS[vif_translator]
     except KeyError:

kuryr_kubernetes/tests/unit/cni/test_binding.py

@@ -13,13 +13,18 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 import mock
+import os
 import uuid
 
+
 from os_vif import objects as osv_objects
+from os_vif.objects import fields as osv_fields
 from oslo_config import cfg
+from oslo_utils import uuidutils
 
 from kuryr_kubernetes.cni.binding import base
 from kuryr_kubernetes.cni.binding import sriov
+from kuryr_kubernetes.cni.binding import vhostuser
 from kuryr_kubernetes import constants as k_const
 from kuryr_kubernetes import exceptions
 from kuryr_kubernetes import objects
@@ -429,3 +434,92 @@ class TestSriovDriver(TestDriverMixin, test_base.TestCase):
         cls._return_device_driver(m_driver, self.vif)
         m_driver._bind_device.assert_called_once_with(pci, old_driver,
                                                       new_driver)
+
+
+class TestVHostUserDriver(TestDriverMixin, test_base.TestCase):
+    def setUp(self):
+        super(TestVHostUserDriver, self).setUp()
+        self.vu_mount_point = '/var/run/cni'
+        self.vu_ovs_path = '/var/run/openvswitch'
+        CONF.set_override('mount_point', self.vu_mount_point,
+                          group='vhostuser')
+        CONF.set_override('ovs_vhu_path', self.vu_ovs_path,
+                          group='vhostuser')
+        self.vif = fake._fake_vif(osv_objects.vif.VIFVHostUser)
+        self.vif.path = self.vu_mount_point
+        self.vif.address = '64:0f:2b:5f:0c:1c'
+        self.port_name = vhostuser._get_vhostuser_port_name(self.vif)
+        self.cont_id = uuidutils.generate_uuid()
+
+    @mock.patch('kuryr_kubernetes.cni.binding.base._need_configure_l3')
+    @mock.patch('kuryr_kubernetes.cni.plugins.k8s_cni_registry.'
+                'K8sCNIRegistryPlugin.report_drivers_health')
+    @mock.patch('os.rename')
+    @mock.patch('os.path.exists', mock.Mock(return_value=True))
+    @mock.patch('kuryr_kubernetes.cni.binding.vhostuser.VIFVHostUserDriver.'
+                '_write_config')
+    @mock.patch('kuryr_kubernetes.cni.binding.vhostuser._check_sock_file')
+    @mock.patch('os_vif.plug')
+    def test_connect_client(self, m_vif_plug, m_check_sock, m_write_conf,
+                            m_os_rename, m_report, m_need_l3):
+        m_need_l3.return_value = False
+        self.vif.mode = osv_fields.VIFVHostUserMode.CLIENT
+        m_check_sock.return_value = True
+        base.connect(self.vif, self.instance_info, self.ifname, self.netns,
+                     m_report, container_id=self.cont_id)
+        vu_dst_socket = os.path.join(self.vu_mount_point, self.port_name)
+        vu_src_socket = os.path.join(self.vu_ovs_path, self.port_name)
+
+        m_vif_plug.assert_called_once_with(self.vif, self.instance_info)
+        m_os_rename.assert_called_once_with(vu_src_socket, vu_dst_socket)
+        m_write_conf.assert_called_once_with(self.cont_id, self.ifname,
+                                             self.port_name, self.vif)
+        m_report.assert_called_once()
+
+    @mock.patch('kuryr_kubernetes.cni.binding.base._need_configure_l3')
+    @mock.patch('kuryr_kubernetes.cni.plugins.k8s_cni_registry.'
+                'K8sCNIRegistryPlugin.report_drivers_health')
+    @mock.patch('kuryr_kubernetes.cni.binding.vhostuser.VIFVHostUserDriver.'
+                '_write_config')
+    @mock.patch('os_vif.plug')
+    def test_connect_server(self, m_vif_plug, m_write_conf,
+                            m_report, m_need_l3):
+        m_need_l3.return_value = False
+        self.vif.mode = osv_fields.VIFVHostUserMode.SERVER
+        base.connect(self.vif, self.instance_info, self.ifname, self.netns,
+                     m_report, container_id=self.cont_id)
+        m_vif_plug.assert_called_once_with(self.vif, self.instance_info)
+        m_write_conf.assert_called_once_with(self.cont_id, self.ifname,
+                                             self.port_name, self.vif)
+        m_report.assert_called_once()
+
+    @mock.patch('kuryr_kubernetes.cni.plugins.k8s_cni_registry.'
+                'K8sCNIRegistryPlugin.report_drivers_health')
+    @mock.patch('kuryr_kubernetes.cni.binding.vhostuser._check_sock_file',
+                mock.Mock(return_value=False))
+    @mock.patch('kuryr_kubernetes.cni.binding.vhostuser.VIFVHostUserDriver.'
+                '_write_config', mock.Mock())
+    @mock.patch('os_vif.plug')
+    def test_connect_nosocket(self, m_vif_plug, m_report):
+        self.vif.mode = osv_fields.VIFVHostUserMode.CLIENT
+        self.assertRaises(exceptions.CNIError, base.connect, self.vif,
+                          self.instance_info, self.ifname, self.netns,
+                          m_report, container_id=self.cont_id)
+
+    @mock.patch('kuryr_kubernetes.cni.plugins.k8s_cni_registry.'
+                'K8sCNIRegistryPlugin.report_drivers_health')
+    @mock.patch('kuryr_kubernetes.cni.binding.vhostuser._get_vhu_sock')
+    @mock.patch('os.remove')
+    @mock.patch('os.path.exists', mock.Mock(return_value=True))
+    @mock.patch('os_vif.unplug')
+    def test_disconnect(self, m_os_unplug, m_os_remove, m_get_vhu_sock,
+                        m_report):
+        m_get_vhu_sock.return_value = self.port_name
+        base.disconnect(self.vif, self.instance_info, self.ifname, self.netns,
+                        m_report, container_id=self.cont_id)
+        conf_file_path = '{}/{}-{}'.format(self.vu_mount_point,
+                                           self.cont_id, self.ifname)
+        vhu_sock_path = '{}/{}'.format(self.vu_mount_point,
+                                       self.port_name)
+        os_remove_calls = [mock.call(vhu_sock_path), mock.call(conf_file_path)]
+        m_os_remove.assert_has_calls(os_remove_calls)

kuryr_kubernetes/tests/unit/test_os_vif_util.py

@@ -184,7 +184,7 @@ class TestOSVIFUtils(test_base.TestCase):
         port_profile = mock.sentinel.port_profile
         network = mock.sentinel.network
         port_active = mock.sentinel.port_active
-        vif_name = mock.sentinel.vif_name
+        vif_name = "vhu01234567-89"
         hybrid_bridge = mock.sentinel.hybrid_bridge
         vif = mock.sentinel.vif
         port = fake.get_port_obj(port_id=port_id,
@@ -240,7 +240,7 @@ class TestOSVIFUtils(test_base.TestCase):
 
         subnets = mock.sentinel.subnets
         network = mock.sentinel.network
-        vif_name = mock.sentinel.vif_name
+        vif_name = "vhu01234567-89"
         vif = mock.sentinel.vif
 
         m_mk_profile.return_value = port.profile
@@ -258,6 +258,92 @@ class TestOSVIFUtils(test_base.TestCase):
         self.assertEqual(network.bridge,
                          port.binding_vif_details['bridge_name'])
 
+    @mock.patch('kuryr_kubernetes.os_vif_util._get_vhu_vif_name')
+    @mock.patch('kuryr_kubernetes.os_vif_util._is_port_active')
+    @mock.patch('kuryr_kubernetes.os_vif_util._make_vif_network')
+    @mock.patch('os_vif.objects.vif.VIFVHostUser')
+    @mock.patch('os_vif.objects.vif.VIFPortProfileOpenVSwitch')
+    def test_neutron_to_osvif_vif_ovs_vu_client(self, m_mk_profile, m_mk_vif,
+                                                m_make_vif_network,
+                                                m_is_port_active,
+                                                m_get_vif_name):
+        vif_plugin = 'vhostuser'
+        o_cfg.CONF.set_override('mount_point',
+                                '/var/lib/cni/vhostuser',
+                                group='vhostuser')
+        port_id = mock.sentinel.port_id
+        mac_address = mock.sentinel.mac_address
+        ovs_bridge = mock.sentinel.ovs_bridge
+        subnets = mock.sentinel.subnets
+        port_profile = mock.sentinel.port_profile
+        network = mock.sentinel.network
+        port_active = mock.sentinel.port_active
+        vif_name = "vhu01234567-89"
+        vif = mock.sentinel.vif
+
+        m_mk_profile.return_value = port_profile
+        m_make_vif_network.return_value = network
+        m_is_port_active.return_value = port_active
+        m_get_vif_name.return_value = vif_name
+        m_mk_vif.return_value = vif
+
+        port = fake.get_port_obj(port_id=port_id,
+                                 vif_details={'ovs_hybrid_plug': False,
+                                              'bridge_name': ovs_bridge,
+                                              'vhostuser_mode': 'client'})
+        port.mac_address = mac_address
+
+        self.assertEqual(vif, ovu.neutron_to_osvif_vif_ovs(vif_plugin, port,
+                                                           subnets))
+        m_mk_profile.assert_called_once_with(interface_id=port_id)
+        m_make_vif_network.assert_called_once_with(port, subnets)
+        m_is_port_active.assert_called_once_with(port)
+        m_get_vif_name.assert_called_once_with(port_id)
+        self.assertEqual(ovs_bridge, network.bridge)
+
+    @mock.patch('kuryr_kubernetes.os_vif_util._get_vhu_vif_name')
+    @mock.patch('kuryr_kubernetes.os_vif_util._is_port_active')
+    @mock.patch('kuryr_kubernetes.os_vif_util._make_vif_network')
+    @mock.patch('os_vif.objects.vif.VIFVHostUser')
+    @mock.patch('os_vif.objects.vif.VIFPortProfileOpenVSwitch')
+    def test_neutron_to_osvif_vif_ovs_vu_server(self, m_mk_profile, m_mk_vif,
+                                                m_make_vif_network,
+                                                m_is_port_active,
+                                                m_get_vif_name):
+        vif_plugin = 'vhostuser'
+        o_cfg.CONF.set_override('mount_point',
+                                '/var/lib/cni/vhostuser',
+                                group='vhostuser')
+        port_id = mock.sentinel.port_id
+        mac_address = mock.sentinel.mac_address
+        ovs_bridge = mock.sentinel.ovs_bridge
+        subnets = mock.sentinel.subnets
+        port_profile = mock.sentinel.port_profile
+        network = mock.sentinel.network
+        port_active = mock.sentinel.port_active
+        vif_name = mock.sentinel.vif_name
+        vif = mock.sentinel.vif
+
+        m_mk_profile.return_value = port_profile
+        m_make_vif_network.return_value = network
+        m_is_port_active.return_value = port_active
+        m_get_vif_name.return_value = vif_name
+        m_mk_vif.return_value = vif
+
+        port = fake.get_port_obj(port_id=port_id,
+                                 vif_details={'ovs_hybrid_plug': False,
+                                              'bridge_name': ovs_bridge,
+                                              'vhostuser_mode': 'server'})
+        port.mac_address = mac_address
+
+        self.assertEqual(vif, ovu.neutron_to_osvif_vif_ovs(vif_plugin, port,
+                                                           subnets))
+        m_mk_profile.assert_called_once_with(interface_id=port_id)
+        m_make_vif_network.assert_called_once_with(port, subnets)
+        m_is_port_active.assert_called_once_with(port)
+        m_get_vif_name.assert_called_once_with(port_id)
+        self.assertEqual(ovs_bridge, network.bridge)
+
     @mock.patch('kuryr_kubernetes.os_vif_util._get_vif_name')
     @mock.patch('kuryr_kubernetes.os_vif_util._is_port_active')
     @mock.patch('kuryr_kubernetes.os_vif_util._make_vif_network')

setup.cfg

@@ -35,11 +35,13 @@ console_scripts =
 kuryr_kubernetes.vif_translators =
     ovs = kuryr_kubernetes.os_vif_util:neutron_to_osvif_vif_ovs
     sriov = kuryr_kubernetes.os_vif_util:neutron_to_osvif_vif_sriov
+    vhostuser = kuryr_kubernetes.os_vif_util:neutron_to_osvif_vif_ovs
 
 kuryr_kubernetes.cni.binding =
     VIFBridge = kuryr_kubernetes.cni.binding.bridge:BridgeDriver
     VIFOpenVSwitch = kuryr_kubernetes.cni.binding.bridge:VIFOpenVSwitchDriver
     VIFDPDKNested = kuryr_kubernetes.cni.binding.dpdk:DpdkDriver
+    VIFVHostUser = kuryr_kubernetes.cni.binding.vhostuser:VIFVHostUserDriver
     VIFVlanNested = kuryr_kubernetes.cni.binding.nested:VlanDriver
     VIFMacvlanNested = kuryr_kubernetes.cni.binding.nested:MacvlanDriver
     VIFSriov = kuryr_kubernetes.cni.binding.sriov:VIFSriovDriver