Add SR-IOV binding driver to CNI
This commit includes a binding driver for SR-IOV interfaces. The driver scans VFs of a PF for each SR-IOV interface requested and assignes them to the Pod. This commit adds new config parameter `physical_device_mappings`. It is similar to neutron-sriov-nic-agent and helps manage PFs and physnets. Implements: blueprint kuryr-kubernetes-sriov-support Change-Id: Icda852cef35efdb75daeae78f7a093fe516f4c02 Signed-off-by: Danil Golov <d.golov@partner.samsung.com> Signed-off-by: Alexey Perevalov <a.perevalov@samsung.com> Signed-off-by: Vladimir Kuramshin <v.kuramshin@samsung.com>
This commit is contained in:
parent
8e60dcc4aa
commit
4d53db2fdf
194
kuryr_kubernetes/cni/binding/sriov.py
Normal file
194
kuryr_kubernetes/cni/binding/sriov.py
Normal file
@ -0,0 +1,194 @@
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import collections
|
||||
import os
|
||||
|
||||
from kuryr.lib._i18n import _
|
||||
from oslo_concurrency import lockutils
|
||||
from oslo_concurrency import processutils
|
||||
from oslo_config import cfg
|
||||
from oslo_log import log as logging
|
||||
|
||||
from kuryr_kubernetes.cni.binding import base as b_base
|
||||
from kuryr_kubernetes import config
|
||||
from kuryr_kubernetes import exceptions
|
||||
from kuryr_kubernetes import utils
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
CONF = cfg.CONF
|
||||
|
||||
|
||||
class VIFSriovDriver(object):
|
||||
def __init__(self):
|
||||
self._lock = None
|
||||
self._device_pf_mapping = self._get_device_pf_mapping()
|
||||
|
||||
def release_lock_object(func):
|
||||
def wrapped(self, *args, **kwargs):
|
||||
try:
|
||||
func(self, *args, **kwargs)
|
||||
finally:
|
||||
if self._lock.acquired:
|
||||
self._lock.release()
|
||||
return
|
||||
return wrapped
|
||||
|
||||
@release_lock_object
|
||||
def connect(self, vif, ifname, netns):
|
||||
physnet = vif.physnet
|
||||
|
||||
h_ipdb = b_base.get_ipdb()
|
||||
c_ipdb = b_base.get_ipdb(netns)
|
||||
|
||||
pf_names = self._get_host_pf_names(physnet)
|
||||
vf_name, vf_index, pf = self._get_available_vf_info(pf_names)
|
||||
|
||||
if not vf_name:
|
||||
error_msg = "No free interfaces for pfysnet {} available".format(
|
||||
physnet)
|
||||
LOG.error(error_msg)
|
||||
raise exceptions.CNIError(error_msg)
|
||||
|
||||
if vif.network.should_provide_vlan:
|
||||
vlan_id = vif.network.vlan
|
||||
self._set_vf_vlan(pf, vf_index, vlan_id)
|
||||
|
||||
with h_ipdb.interfaces[vf_name] as host_iface:
|
||||
host_iface.net_ns_fd = utils.convert_netns(netns)
|
||||
|
||||
with c_ipdb.interfaces[vf_name] as iface:
|
||||
iface.ifname = ifname
|
||||
iface.address = vif.address
|
||||
iface.mtu = vif.network.mtu
|
||||
iface.up()
|
||||
|
||||
def disconnect(self, vif, ifname, netns):
|
||||
# NOTE(k.zaitsev): when netns is deleted the interface is
|
||||
# returned automatically to host netns. We may reset
|
||||
# it to all-zero state
|
||||
pass
|
||||
|
||||
def _get_host_pf_names(self, physnet):
|
||||
"""Return a list of PFs, that belong to a physnet"""
|
||||
|
||||
if physnet not in self._device_pf_mapping:
|
||||
raise cfg.Error(
|
||||
"No mapping for physnet {} in {}".format(
|
||||
physnet, self._device_pf_mapping))
|
||||
return self._device_pf_mapping[physnet]
|
||||
|
||||
def _get_available_vf_info(self, pf_names):
|
||||
"""Scan /sys for unacquired VF among PFs in pf_names"""
|
||||
|
||||
for pf in pf_names:
|
||||
pf_sys_path = '/sys/class/net/{}/device'.format(pf)
|
||||
nvfs = self._get_total_vfs(pf)
|
||||
for vf_index in range(nvfs):
|
||||
vf_sys_path = os.path.join(pf_sys_path,
|
||||
'virtfn{}'.format(vf_index),
|
||||
'net')
|
||||
# TODO(kzaitsev): use /var/run/kuryr/smth
|
||||
lock_path = os.path.join("/tmp",
|
||||
"{}.{}".format(pf, vf_index))
|
||||
self._acquire(lock_path)
|
||||
LOG.debug("Aquired %s lock", lock_path)
|
||||
try:
|
||||
vf_names = os.listdir(vf_sys_path)
|
||||
except OSError:
|
||||
LOG.debug("Could not open %s. "
|
||||
"Skipping vf %s for pf %s", vf_sys_path,
|
||||
vf_index, pf)
|
||||
self._release()
|
||||
continue
|
||||
if not vf_names:
|
||||
LOG.debug("No interfaces in %s"
|
||||
"Skipping vf %s for pf %s", vf_sys_path,
|
||||
vf_index, pf)
|
||||
self._release()
|
||||
continue
|
||||
vf_name = vf_names[0]
|
||||
LOG.debug("Aquiring vf %s of pf %s", vf_index, pf)
|
||||
return vf_name, vf_index, pf
|
||||
return None, None, None
|
||||
|
||||
def _acquire(self, path):
|
||||
if self._lock and self._lock.acquired:
|
||||
raise RuntimeError(_("Attempting to lock {} when {} "
|
||||
"is already locked.").format(path, self._lock))
|
||||
self._lock = lockutils.InterProcessLock(path=path)
|
||||
return self._lock.acquire()
|
||||
|
||||
def _release(self):
|
||||
if not self._lock:
|
||||
raise RuntimeError(_("Attempting release an empty lock"))
|
||||
return self._lock.release()
|
||||
|
||||
def _get_total_vfs(self, pf):
|
||||
"""Read /sys information for configured number of VFs of a PF"""
|
||||
|
||||
pf_sys_path = '/sys/class/net/{}/device'.format(pf)
|
||||
total_fname = os.path.join(pf_sys_path, 'sriov_numvfs')
|
||||
try:
|
||||
with open(total_fname) as total_f:
|
||||
data = total_f.read()
|
||||
except IOError:
|
||||
LOG.warning("Could not open %s. No VFs for %s", total_fname, pf)
|
||||
return 0
|
||||
nvfs = 0
|
||||
try:
|
||||
nvfs = int(data.strip())
|
||||
except ValueError:
|
||||
LOG.warning("Could not parse %s from %s. No VFs for %s", data,
|
||||
total_fname, pf)
|
||||
return 0
|
||||
LOG.debug("PF %s has %s VFs", pf, nvfs)
|
||||
return nvfs
|
||||
|
||||
def _set_vf_vlan(self, pf, vf_index, vlan_id):
|
||||
"""Call `ip link set enp1s0f0 vf 5 vlan 10`"""
|
||||
cmd = [
|
||||
'ip', 'link',
|
||||
'set', pf, 'vf', vf_index, 'vlan', vlan_id
|
||||
]
|
||||
try:
|
||||
return processutils.execute(*cmd, run_as_root=True)
|
||||
except Exception:
|
||||
LOG.exception("Unable to execute %s", cmd)
|
||||
raise
|
||||
|
||||
def is_healthy(self):
|
||||
bridge_name = CONF.neutron_defaults.ovs_bridge
|
||||
try:
|
||||
with b_base.get_ipdb() as h_ipdb:
|
||||
h_ipdb.interfaces[bridge_name]
|
||||
return True
|
||||
except Exception:
|
||||
LOG.warning("Default OVS bridge %s does not exist on "
|
||||
"the host.", bridge_name)
|
||||
return False
|
||||
|
||||
def _get_device_pf_mapping(self):
|
||||
"""Return a mapping in format {<physnet_name>:[<pf_name>, ...]}"""
|
||||
|
||||
phys_mappings = config.CONF.sriov.physical_device_mappings
|
||||
physnets = collections.defaultdict(list)
|
||||
for phys_map in phys_mappings:
|
||||
try:
|
||||
netname, ifname = phys_map.split(':', 1)
|
||||
except ValueError:
|
||||
raise cfg.Error(
|
||||
"Invalid mapping {}".format(phys_map))
|
||||
physnets[netname].append(ifname)
|
||||
return physnets
|
@ -238,11 +238,19 @@ nested_vif_driver_opts = [
|
||||
]
|
||||
|
||||
DEFAULT_PHYSNET_SUBNET_MAPPINGS = {}
|
||||
DEFAULT_DEVICE_MAPPINGS = []
|
||||
sriov_opts = [
|
||||
cfg.DictOpt('default_physnet_subnets',
|
||||
help=_("A mapping of default subnets for certain physnets "
|
||||
"in a form of physnet-name:<SUBNET-ID>"),
|
||||
default=DEFAULT_PHYSNET_SUBNET_MAPPINGS),
|
||||
cfg.ListOpt('physical_device_mappings',
|
||||
default=DEFAULT_DEVICE_MAPPINGS,
|
||||
help=_("Comma-separated list of "
|
||||
"<physical_network>:<network_device> tuples mapping "
|
||||
"physical network names to the agent's node-specific "
|
||||
"physical network device interfaces of SR-IOV physical "
|
||||
"function to be used for VLAN networks.")),
|
||||
]
|
||||
|
||||
|
||||
|
@ -41,6 +41,7 @@ kuryr_kubernetes.cni.binding =
|
||||
VIFOpenVSwitch = kuryr_kubernetes.cni.binding.bridge:VIFOpenVSwitchDriver
|
||||
VIFVlanNested = kuryr_kubernetes.cni.binding.nested:VlanDriver
|
||||
VIFMacvlanNested = kuryr_kubernetes.cni.binding.nested:MacvlanDriver
|
||||
VIFSriov = kuryr_kubernetes.cni.binding.sriov:VIFSriovDriver
|
||||
|
||||
kuryr_kubernetes.controller.drivers.pod_project =
|
||||
default = kuryr_kubernetes.controller.drivers.default_project:DefaultPodProjectDriver
|
||||
|
Loading…
Reference in New Issue
Block a user