Browse Source

Merge "Support DPDK application on bare-metal host"

changes/25/713625/1
Zuul 2 years ago
committed by Gerrit Code Review
parent
commit
78846f7048
  1. 138
      kuryr_kubernetes/cni/binding/vhostuser.py
  2. 12
      kuryr_kubernetes/config.py
  3. 1
      kuryr_kubernetes/controller/drivers/vif_pool.py
  4. 40
      kuryr_kubernetes/os_vif_util.py
  5. 94
      kuryr_kubernetes/tests/unit/cni/test_binding.py
  6. 90
      kuryr_kubernetes/tests/unit/test_os_vif_util.py
  7. 2
      setup.cfg

138
kuryr_kubernetes/cni/binding/vhostuser.py

@ -0,0 +1,138 @@
# Copyright (c) 2020 Samsung Electronics Co., Ltd.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os.path
import stat
from kuryr.lib._i18n import _
from os_vif.objects import fields as osv_fields
from oslo_config import cfg
from oslo_log import log
from oslo_serialization import jsonutils
from vif_plug_ovs import constants
from vif_plug_ovs import ovs
from kuryr_kubernetes.cni.binding import base
from kuryr_kubernetes import config
from kuryr_kubernetes import exceptions as k_exc
from kuryr_kubernetes.handlers import health
LOG = log.getLogger(__name__)
def _get_vhostport_type(vif):
if vif.mode == osv_fields.VIFVHostUserMode.SERVER:
return 'dpdkvhostuserclient'
elif vif.mode == osv_fields.VIFVHostUserMode.CLIENT:
return 'dpdkvhostuser'
raise k_exc.IntegrityError(
_("Unknown vhostuser mode %(mode)s for vif %(vif_id)s")
% {'mode': vif.mode, 'vif_id': vif.id})
def _get_vhostuser_port_name(vif):
return ovs.OvsPlugin.gen_port_name(constants.OVS_VHOSTUSER_PREFIX, vif.id)
def _get_vhu_sock(config_file_path):
with open(config_file_path, 'r') as f:
conf = jsonutils.load(f)
return conf['vhostname']
def _check_sock_file(vhostuser_socket):
mode = os.stat(vhostuser_socket).st_mode
return stat.S_ISSOCK(mode)
class VIFVHostUserDriver(health.HealthHandler, base.BaseBindingDriver):
def __init__(self):
super(VIFVHostUserDriver, self).__init__()
self.mount_path = config.CONF.vhostuser.mount_point
self.ovs_vu_path = config.CONF.vhostuser.ovs_vhu_path
if not self.mount_path:
raise cfg.RequiredOptError('mount_point', 'vhostuser')
def _write_config(self, container_id, ifname, port_name, vif):
"""Write vhostuser configuration file
This function writes configuration file, this file will be used by
application inside container and for cleanup (def disconnect)
procedure.
"""
vhost_conf = {}
vhost_conf["vhostname"] = port_name
vhost_conf["vhostmac"] = vif.address
vhost_conf["mode"] = vif.mode
with open(self._config_file_path(container_id, ifname), "w") as f:
jsonutils.dump(vhost_conf, f)
def _config_file_path(self, container_id, ifname):
return os.path.join(self.mount_path, f'{container_id}-{ifname}')
def connect(self, vif, ifname, netns, container_id):
port_name = _get_vhostuser_port_name(vif)
self._write_config(container_id, ifname, port_name, vif)
# no need to copy in case of SERVER mode
if vif.mode == osv_fields.VIFVHostUserMode.SERVER:
return
src_vhu_sock = os.path.join(self.ovs_vu_path, port_name)
if _check_sock_file(src_vhu_sock):
dst_vhu_sock = os.path.join(vif.path, port_name)
LOG.debug("Moving %s to %s while processing VIF %s", src_vhu_sock,
dst_vhu_sock, vif.id)
os.rename(src_vhu_sock, dst_vhu_sock)
else:
error_msg = "Socket %s required for VIF %s doesn't exist".format(
src_vhu_sock, vif.id)
LOG.error(error_msg)
raise k_exc.CNIError(error_msg)
def disconnect(self, vif, ifname, netns, container_id):
# This function removes configuration file and appropriate
# socket file. Unfortunatelly Open vSwitch daemon can't remove
# moved socket, so we have to do it
config_file_path = self._config_file_path(container_id, ifname)
if not os.path.exists(config_file_path):
LOG.warning("Configuration file: %s for VIF %s doesn't exist!",
config_file_path, vif.id)
return
vhu_sock_path = os.path.join(self.mount_path,
_get_vhu_sock(config_file_path))
LOG.debug("remove: %s, %s", config_file_path, vhu_sock_path)
os.remove(vhu_sock_path)
os.remove(config_file_path)
def is_alive(self):
healthy = False
try:
healthy = (os.path.exists(self.ovs_vu_path)
and os.path.exists(self.mount_path))
except Exception:
LOG.exception('Error when determining health status of vhostuser '
'CNI driver.')
if not healthy:
LOG.error('Directory %s or %s does not exist or Kuryr has no '
'permissions to access it. Marking vhostuser binding '
'driver as unhealthy.', self.ovs_vu_path,
self.mount_path)
return healthy

12
kuryr_kubernetes/config.py

@ -319,6 +319,17 @@ sriov_opts = [
]
vhostuser = [
cfg.StrOpt('mount_point',
help=_("Path where vhost-user port will be created "
"also it should be mount point for pod"),
default='/var/cni/vhostuser'),
cfg.StrOpt('ovs_vhu_path',
help=_("Path where OVS keeps socket files for vhost-user "
"ports"),
default='/var/run/openvswitch/')
]
CONF = cfg.CONF
CONF.register_opts(kuryr_k8s_opts)
CONF.register_opts(daemon_opts, group='cni_daemon')
@ -328,6 +339,7 @@ CONF.register_opts(octavia_defaults, group='octavia_defaults')
CONF.register_opts(cache_defaults, group='cache_defaults')
CONF.register_opts(nested_vif_driver_opts, group='pod_vif_nested')
CONF.register_opts(sriov_opts, group='sriov')
CONF.register_opts(vhostuser, group='vhostuser')
CONF.register_opts(lib_config.core_opts)
CONF.register_opts(lib_config.binding_opts, 'binding')

1
kuryr_kubernetes/controller/drivers/vif_pool.py

@ -100,6 +100,7 @@ VIF_TYPE_TO_DRIVER_MAPPING = {
'VIFMacvlanNested': 'nested-macvlan',
'VIFSriov': 'sriov',
'VIFDPDKNested': 'nested-dpdk',
'VIFVHostUser': 'neutron-vif',
}

40
kuryr_kubernetes/os_vif_util.py

@ -23,7 +23,9 @@ from os_vif.objects import route as osv_route
from os_vif.objects import subnet as osv_subnet
from os_vif.objects import vif as osv_vif
from oslo_config import cfg as oslo_cfg
from oslo_log import log as logging
from stevedore import driver as stv_driver
from vif_plug_ovs import constants as osv_const
from kuryr_kubernetes import config
from kuryr_kubernetes import constants as const
@ -31,6 +33,8 @@ from kuryr_kubernetes import exceptions as k_exc
from kuryr_kubernetes.objects import vif as k_vif
LOG = logging.getLogger(__name__)
# REVISIT(ivc): consider making this module part of kuryr-lib
_VIF_TRANSLATOR_NAMESPACE = "kuryr_kubernetes.vif_translators"
_VIF_MANAGERS = {}
@ -193,6 +197,15 @@ def _make_vif_network(neutron_port, subnets):
return network
# TODO(a.perevalov) generalize it with get_veth_pair_names
# but it's reasonable if we're going to add vhostuser support
# into kuryr project
def _get_vhu_vif_name(port_id):
ifname = osv_const.OVS_VHOSTUSER_PREFIX + port_id
ifname = ifname[:kl_const.NIC_NAME_LEN]
return ifname
def _get_vif_name(neutron_port):
"""Gets a VIF device name for port.
@ -249,8 +262,30 @@ def neutron_to_osvif_vif_ovs(vif_plugin, os_port, subnets):
network = _make_vif_network(os_port, subnets)
network.bridge = ovs_bridge
if details.get('ovs_hybrid_plug'):
vhostuser_mode = details.get('vhostuser_mode', False)
LOG.debug('Detected vhostuser_mode=%s for port %s', vhostuser_mode,
os_port.id)
if vhostuser_mode:
# TODO(a.perevalov) obtain path to mount point from pod's mountVolumes
vhostuser_mount_point = (config.CONF.vhostuser.mount_point)
if not vhostuser_mount_point:
raise oslo_cfg.RequiredOptError('vhostuser_mount_point',
'neutron_defaults')
vif = osv_vif.VIFVHostUser(
id=os_port.id,
address=os_port.mac_address,
network=network,
has_traffic_filtering=details.get('port_filter', False),
preserve_on_delete=False,
active=_is_port_active(os_port),
port_profile=profile,
plugin='ovs',
path=vhostuser_mount_point,
mode=vhostuser_mode,
vif_name=_get_vhu_vif_name(os_port.id),
bridge_name=network.bridge)
elif details.get('ovs_hybrid_plug'):
vif = osv_vif.VIFBridge(
id=os_port.id,
address=os_port.mac_address,
@ -386,7 +421,6 @@ def neutron_to_osvif_vif(vif_translator, os_port, subnets):
:param subnets: subnet mapping as returned by PodSubnetsDriver.get_subnets
:return: os-vif VIF object
"""
try:
mgr = _VIF_MANAGERS[vif_translator]
except KeyError:

94
kuryr_kubernetes/tests/unit/cni/test_binding.py

@ -13,13 +13,18 @@
# License for the specific language governing permissions and limitations
# under the License.
import mock
import os
import uuid
from os_vif import objects as osv_objects
from os_vif.objects import fields as osv_fields
from oslo_config import cfg
from oslo_utils import uuidutils
from kuryr_kubernetes.cni.binding import base
from kuryr_kubernetes.cni.binding import sriov
from kuryr_kubernetes.cni.binding import vhostuser
from kuryr_kubernetes import constants as k_const
from kuryr_kubernetes import exceptions
from kuryr_kubernetes import objects
@ -429,3 +434,92 @@ class TestSriovDriver(TestDriverMixin, test_base.TestCase):
cls._return_device_driver(m_driver, self.vif)
m_driver._bind_device.assert_called_once_with(pci, old_driver,
new_driver)
class TestVHostUserDriver(TestDriverMixin, test_base.TestCase):
def setUp(self):
super(TestVHostUserDriver, self).setUp()
self.vu_mount_point = '/var/run/cni'
self.vu_ovs_path = '/var/run/openvswitch'
CONF.set_override('mount_point', self.vu_mount_point,
group='vhostuser')
CONF.set_override('ovs_vhu_path', self.vu_ovs_path,
group='vhostuser')
self.vif = fake._fake_vif(osv_objects.vif.VIFVHostUser)
self.vif.path = self.vu_mount_point
self.vif.address = '64:0f:2b:5f:0c:1c'
self.port_name = vhostuser._get_vhostuser_port_name(self.vif)
self.cont_id = uuidutils.generate_uuid()
@mock.patch('kuryr_kubernetes.cni.binding.base._need_configure_l3')
@mock.patch('kuryr_kubernetes.cni.plugins.k8s_cni_registry.'
'K8sCNIRegistryPlugin.report_drivers_health')
@mock.patch('os.rename')
@mock.patch('os.path.exists', mock.Mock(return_value=True))
@mock.patch('kuryr_kubernetes.cni.binding.vhostuser.VIFVHostUserDriver.'
'_write_config')
@mock.patch('kuryr_kubernetes.cni.binding.vhostuser._check_sock_file')
@mock.patch('os_vif.plug')
def test_connect_client(self, m_vif_plug, m_check_sock, m_write_conf,
m_os_rename, m_report, m_need_l3):
m_need_l3.return_value = False
self.vif.mode = osv_fields.VIFVHostUserMode.CLIENT
m_check_sock.return_value = True
base.connect(self.vif, self.instance_info, self.ifname, self.netns,
m_report, container_id=self.cont_id)
vu_dst_socket = os.path.join(self.vu_mount_point, self.port_name)
vu_src_socket = os.path.join(self.vu_ovs_path, self.port_name)
m_vif_plug.assert_called_once_with(self.vif, self.instance_info)
m_os_rename.assert_called_once_with(vu_src_socket, vu_dst_socket)
m_write_conf.assert_called_once_with(self.cont_id, self.ifname,
self.port_name, self.vif)
m_report.assert_called_once()
@mock.patch('kuryr_kubernetes.cni.binding.base._need_configure_l3')
@mock.patch('kuryr_kubernetes.cni.plugins.k8s_cni_registry.'
'K8sCNIRegistryPlugin.report_drivers_health')
@mock.patch('kuryr_kubernetes.cni.binding.vhostuser.VIFVHostUserDriver.'
'_write_config')
@mock.patch('os_vif.plug')
def test_connect_server(self, m_vif_plug, m_write_conf,
m_report, m_need_l3):
m_need_l3.return_value = False
self.vif.mode = osv_fields.VIFVHostUserMode.SERVER
base.connect(self.vif, self.instance_info, self.ifname, self.netns,
m_report, container_id=self.cont_id)
m_vif_plug.assert_called_once_with(self.vif, self.instance_info)
m_write_conf.assert_called_once_with(self.cont_id, self.ifname,
self.port_name, self.vif)
m_report.assert_called_once()
@mock.patch('kuryr_kubernetes.cni.plugins.k8s_cni_registry.'
'K8sCNIRegistryPlugin.report_drivers_health')
@mock.patch('kuryr_kubernetes.cni.binding.vhostuser._check_sock_file',
mock.Mock(return_value=False))
@mock.patch('kuryr_kubernetes.cni.binding.vhostuser.VIFVHostUserDriver.'
'_write_config', mock.Mock())
@mock.patch('os_vif.plug')
def test_connect_nosocket(self, m_vif_plug, m_report):
self.vif.mode = osv_fields.VIFVHostUserMode.CLIENT
self.assertRaises(exceptions.CNIError, base.connect, self.vif,
self.instance_info, self.ifname, self.netns,
m_report, container_id=self.cont_id)
@mock.patch('kuryr_kubernetes.cni.plugins.k8s_cni_registry.'
'K8sCNIRegistryPlugin.report_drivers_health')
@mock.patch('kuryr_kubernetes.cni.binding.vhostuser._get_vhu_sock')
@mock.patch('os.remove')
@mock.patch('os.path.exists', mock.Mock(return_value=True))
@mock.patch('os_vif.unplug')
def test_disconnect(self, m_os_unplug, m_os_remove, m_get_vhu_sock,
m_report):
m_get_vhu_sock.return_value = self.port_name
base.disconnect(self.vif, self.instance_info, self.ifname, self.netns,
m_report, container_id=self.cont_id)
conf_file_path = '{}/{}-{}'.format(self.vu_mount_point,
self.cont_id, self.ifname)
vhu_sock_path = '{}/{}'.format(self.vu_mount_point,
self.port_name)
os_remove_calls = [mock.call(vhu_sock_path), mock.call(conf_file_path)]
m_os_remove.assert_has_calls(os_remove_calls)

90
kuryr_kubernetes/tests/unit/test_os_vif_util.py

@ -184,7 +184,7 @@ class TestOSVIFUtils(test_base.TestCase):
port_profile = mock.sentinel.port_profile
network = mock.sentinel.network
port_active = mock.sentinel.port_active
vif_name = mock.sentinel.vif_name
vif_name = "vhu01234567-89"
hybrid_bridge = mock.sentinel.hybrid_bridge
vif = mock.sentinel.vif
port = fake.get_port_obj(port_id=port_id,
@ -240,7 +240,7 @@ class TestOSVIFUtils(test_base.TestCase):
subnets = mock.sentinel.subnets
network = mock.sentinel.network
vif_name = mock.sentinel.vif_name
vif_name = "vhu01234567-89"
vif = mock.sentinel.vif
m_mk_profile.return_value = port.profile
@ -258,6 +258,92 @@ class TestOSVIFUtils(test_base.TestCase):
self.assertEqual(network.bridge,
port.binding_vif_details['bridge_name'])
@mock.patch('kuryr_kubernetes.os_vif_util._get_vhu_vif_name')
@mock.patch('kuryr_kubernetes.os_vif_util._is_port_active')
@mock.patch('kuryr_kubernetes.os_vif_util._make_vif_network')
@mock.patch('os_vif.objects.vif.VIFVHostUser')
@mock.patch('os_vif.objects.vif.VIFPortProfileOpenVSwitch')
def test_neutron_to_osvif_vif_ovs_vu_client(self, m_mk_profile, m_mk_vif,
m_make_vif_network,
m_is_port_active,
m_get_vif_name):
vif_plugin = 'vhostuser'
o_cfg.CONF.set_override('mount_point',
'/var/lib/cni/vhostuser',
group='vhostuser')
port_id = mock.sentinel.port_id
mac_address = mock.sentinel.mac_address
ovs_bridge = mock.sentinel.ovs_bridge
subnets = mock.sentinel.subnets
port_profile = mock.sentinel.port_profile
network = mock.sentinel.network
port_active = mock.sentinel.port_active
vif_name = "vhu01234567-89"
vif = mock.sentinel.vif
m_mk_profile.return_value = port_profile
m_make_vif_network.return_value = network
m_is_port_active.return_value = port_active
m_get_vif_name.return_value = vif_name
m_mk_vif.return_value = vif
port = fake.get_port_obj(port_id=port_id,
vif_details={'ovs_hybrid_plug': False,
'bridge_name': ovs_bridge,
'vhostuser_mode': 'client'})
port.mac_address = mac_address
self.assertEqual(vif, ovu.neutron_to_osvif_vif_ovs(vif_plugin, port,
subnets))
m_mk_profile.assert_called_once_with(interface_id=port_id)
m_make_vif_network.assert_called_once_with(port, subnets)
m_is_port_active.assert_called_once_with(port)
m_get_vif_name.assert_called_once_with(port_id)
self.assertEqual(ovs_bridge, network.bridge)
@mock.patch('kuryr_kubernetes.os_vif_util._get_vhu_vif_name')
@mock.patch('kuryr_kubernetes.os_vif_util._is_port_active')
@mock.patch('kuryr_kubernetes.os_vif_util._make_vif_network')
@mock.patch('os_vif.objects.vif.VIFVHostUser')
@mock.patch('os_vif.objects.vif.VIFPortProfileOpenVSwitch')
def test_neutron_to_osvif_vif_ovs_vu_server(self, m_mk_profile, m_mk_vif,
m_make_vif_network,
m_is_port_active,
m_get_vif_name):
vif_plugin = 'vhostuser'
o_cfg.CONF.set_override('mount_point',
'/var/lib/cni/vhostuser',
group='vhostuser')
port_id = mock.sentinel.port_id
mac_address = mock.sentinel.mac_address
ovs_bridge = mock.sentinel.ovs_bridge
subnets = mock.sentinel.subnets
port_profile = mock.sentinel.port_profile
network = mock.sentinel.network
port_active = mock.sentinel.port_active
vif_name = mock.sentinel.vif_name
vif = mock.sentinel.vif
m_mk_profile.return_value = port_profile
m_make_vif_network.return_value = network
m_is_port_active.return_value = port_active
m_get_vif_name.return_value = vif_name
m_mk_vif.return_value = vif
port = fake.get_port_obj(port_id=port_id,
vif_details={'ovs_hybrid_plug': False,
'bridge_name': ovs_bridge,
'vhostuser_mode': 'server'})
port.mac_address = mac_address
self.assertEqual(vif, ovu.neutron_to_osvif_vif_ovs(vif_plugin, port,
subnets))
m_mk_profile.assert_called_once_with(interface_id=port_id)
m_make_vif_network.assert_called_once_with(port, subnets)
m_is_port_active.assert_called_once_with(port)
m_get_vif_name.assert_called_once_with(port_id)
self.assertEqual(ovs_bridge, network.bridge)
@mock.patch('kuryr_kubernetes.os_vif_util._get_vif_name')
@mock.patch('kuryr_kubernetes.os_vif_util._is_port_active')
@mock.patch('kuryr_kubernetes.os_vif_util._make_vif_network')

2
setup.cfg

@ -35,11 +35,13 @@ console_scripts =
kuryr_kubernetes.vif_translators =
ovs = kuryr_kubernetes.os_vif_util:neutron_to_osvif_vif_ovs
sriov = kuryr_kubernetes.os_vif_util:neutron_to_osvif_vif_sriov
vhostuser = kuryr_kubernetes.os_vif_util:neutron_to_osvif_vif_ovs
kuryr_kubernetes.cni.binding =
VIFBridge = kuryr_kubernetes.cni.binding.bridge:BridgeDriver
VIFOpenVSwitch = kuryr_kubernetes.cni.binding.bridge:VIFOpenVSwitchDriver
VIFDPDKNested = kuryr_kubernetes.cni.binding.dpdk:DpdkDriver
VIFVHostUser = kuryr_kubernetes.cni.binding.vhostuser:VIFVHostUserDriver
VIFVlanNested = kuryr_kubernetes.cni.binding.nested:VlanDriver
VIFMacvlanNested = kuryr_kubernetes.cni.binding.nested:MacvlanDriver
VIFSriov = kuryr_kubernetes.cni.binding.sriov:VIFSriovDriver

Loading…
Cancel
Save