[WIP] Add SR-IOV binding driver to CNI
This commit includes a binding driver for SR-IOV interfaces. The driver scans VFs of a PF for each SR-IOV interface requested and assignes them to the Pod. This commit adds new config parameter `physical_device_mappings`. It is similar to neutron-sriov-nic-agent and helps manage PFs and physnets. Change-Id: Icda852cef35efdb75daeae78f7a093fe516f4c02 Signed-off-by: Danil Golov <d.golov@partner.samsung.com>
This commit is contained in:
Danil Golov
parent
d7528a02dc
commit
e6b419109b
|
|
@ -0,0 +1,166 @@
|
|||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from kuryr.lib._i18n import _
|
||||
from oslo_concurrency import lockutils
|
||||
from oslo_concurrency import processutils
|
||||
from oslo_config import cfg
|
||||
from oslo_log import log as logging
|
||||
|
||||
from kuryr_kubernetes.cni.binding import base as b_base
|
||||
from kuryr_kubernetes import config
|
||||
from kuryr_kubernetes import exceptions
|
||||
|
||||
import collections
|
||||
import os
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class VIFSRIOVDriver(object):
|
||||
def __init__(self):
|
||||
self._lock = None
|
||||
|
||||
def connect(self, vif, ifname, netns):
|
||||
physnet = vif.physnet
|
||||
|
||||
h_ipdb = b_base.get_ipdb()
|
||||
c_ipdb = b_base.get_ipdb(netns)
|
||||
|
||||
pf_names = self._get_host_pf_names(physnet)
|
||||
vf_name, vf_index, pf = self._get_available_vf_info(pf_names)
|
||||
|
||||
if not vf_name:
|
||||
error_msg = "No free interfaces for pfysnet {} available".format(
|
||||
physnet)
|
||||
LOG.error(error_msg)
|
||||
raise exceptions.CNIError(error_msg)
|
||||
|
||||
vlan_id = vif.network.vlan
|
||||
self._set_vf_vlan(pf, vf_index, vlan_id)
|
||||
|
||||
with h_ipdb.interfaces[vf_name] as host_iface:
|
||||
host_iface.net_ns_fd = netns
|
||||
|
||||
with c_ipdb.interfaces[vf_name] as iface:
|
||||
iface.ifname = ifname
|
||||
iface.address = vif.address
|
||||
iface.mtu = vif.network.mtu
|
||||
iface.up()
|
||||
self._release()
|
||||
|
||||
def disconnect(self, vif, ifname, netns):
|
||||
# NOTE(k.zaitsev): when netns is deleted the interface is
|
||||
# returned automatically to host netns. We may reset
|
||||
# it to all-zero state
|
||||
pass
|
||||
|
||||
def _get_host_pf_names(self, physnet):
|
||||
"""Return a dict of PFs, that belong to a physnet"""
|
||||
|
||||
phys_mappings = config.CONF.sriov.physical_device_mappings
|
||||
physnets = collections.defaultdict(list)
|
||||
for phys_map in phys_mappings:
|
||||
try:
|
||||
netname, ifname = phys_map.split(':', 1)
|
||||
except ValueError:
|
||||
raise cfg.Error(
|
||||
"Invalid mapping {}".format(phys_map))
|
||||
physnets[netname].append(ifname)
|
||||
if physnet not in physnets:
|
||||
raise cfg.Error(
|
||||
"No mapping for physnet {} in {}".format(
|
||||
physnet, phys_mappings))
|
||||
return physnets[physnet]
|
||||
|
||||
def _get_available_vf_info(self, pf_names):
|
||||
"""Scan /sys for unacquired VF among PFs in pf_names"""
|
||||
|
||||
for pf in pf_names:
|
||||
pf_sys_path = '/sys/class/net/{}/device'.format(pf)
|
||||
nvfs = self._get_total_vfs(pf)
|
||||
for vf_index in range(nvfs):
|
||||
vf_sys_path = os.path.join(pf_sys_path,
|
||||
'virtfn{}'.format(vf_index),
|
||||
'net')
|
||||
# TODO(kzaitsev): use /var/run/kuryr/smth
|
||||
lock_path = os.path.join("/tmp",
|
||||
"{}.{}".format(pf, vf_index))
|
||||
self._acquire(lock_path)
|
||||
LOG.debug("Aquired {} lock".format(lock_path))
|
||||
try:
|
||||
vf_names = os.listdir(vf_sys_path)
|
||||
except OSError:
|
||||
LOG.debug("Could not open {}. "
|
||||
"Skipping vf {} for pf {}".format(
|
||||
vf_sys_path, vf_index, pf))
|
||||
self._release()
|
||||
continue
|
||||
if not vf_names:
|
||||
LOG.debug("No interfaces in {}"
|
||||
"Skipping vf {} for pf {}".format(
|
||||
vf_sys_path, vf_index, pf))
|
||||
self._release()
|
||||
continue
|
||||
vf_name = vf_names[0]
|
||||
LOG.debug("Aquiring vf {} of pf {}".format(
|
||||
vf_index, pf))
|
||||
return vf_name, vf_index, pf
|
||||
return None, None, None
|
||||
|
||||
def _acquire(self, path):
|
||||
if self._lock and self._lock.acquired:
|
||||
raise RuntimeError(_("Attempting to lock {} when {} "
|
||||
"is already locked.").format(path, self._lock))
|
||||
self._lock = lockutils.InterProcessLock(path=path)
|
||||
return self._lock.acquire()
|
||||
|
||||
def _release(self):
|
||||
if not self._lock:
|
||||
raise RuntimeError(_("Attempting release an empty lock"))
|
||||
return self._lock.release()
|
||||
|
||||
def _get_total_vfs(self, pf):
|
||||
"""Read /sys information for configured number of VFs of a PF"""
|
||||
|
||||
pf_sys_path = '/sys/class/net/{}/device'.format(pf)
|
||||
total_fname = os.path.join(pf_sys_path, 'sriov_numvfs')
|
||||
try:
|
||||
with open(total_fname) as total_f:
|
||||
data = total_f.read()
|
||||
except IOError:
|
||||
LOG.warning("Could not open {}. No VFs for {}".format(
|
||||
total_fname, pf))
|
||||
return 0
|
||||
nvfs = 0
|
||||
try:
|
||||
nvfs = int(data.strip())
|
||||
except ValueError:
|
||||
LOG.warning("Could parse {} from {}. No VFs for {}".format(
|
||||
data, total_fname, pf))
|
||||
return 0
|
||||
LOG.debug("PF {} has {} VFs".format(pf, nvfs))
|
||||
return nvfs
|
||||
|
||||
def _set_vf_vlan(self, pf, vf_index, vlan_id):
|
||||
"""Call `ip link set enp1s0f0 vf 5 vlan 10`"""
|
||||
cmd = [
|
||||
'ip', 'link',
|
||||
'set', pf, 'vf', vf_index, 'vlan', vlan_id
|
||||
]
|
||||
try:
|
||||
return processutils.execute(*cmd, run_as_root=True)
|
||||
except Exception:
|
||||
LOG.exception("Unable to execute {}".format(cmd))
|
||||
raise
|
||||
|
|
@ -106,7 +106,6 @@ k8s_opts = [
|
|||
|
||||
]
|
||||
|
||||
|
||||
neutron_defaults = [
|
||||
cfg.StrOpt('project',
|
||||
help=_("Default OpenStack project ID for "
|
||||
|
|
@ -134,11 +133,19 @@ octavia_defaults = [
|
|||
|
||||
|
||||
DEFAULT_PHYSNET_SUBNET_MAPPINGS = {}
|
||||
DEFAULT_DEVICE_MAPPINGS = []
|
||||
sriov_opts = [
|
||||
cfg.DictOpt('default_physnet_subnets',
|
||||
help=_("A mapping of default subnets for certain physnets "
|
||||
"in a form of physnet-name:<SUBNET-ID>"),
|
||||
default=DEFAULT_PHYSNET_SUBNET_MAPPINGS),
|
||||
cfg.ListOpt('physical_device_mappings',
|
||||
default=DEFAULT_DEVICE_MAPPINGS,
|
||||
help=_("Comma-separated list of "
|
||||
"<physical_network>:<network_device> tuples mapping "
|
||||
"physical network names to the agent's node-specific "
|
||||
"physical network device interfaces of SR-IOV physical "
|
||||
"function to be used for VLAN networks.")),
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ kuryr_kubernetes.cni.binding =
|
|||
VIFOpenVSwitch = kuryr_kubernetes.cni.binding.bridge:VIFOpenVSwitchDriver
|
||||
VIFVlanNested = kuryr_kubernetes.cni.binding.nested:VlanDriver
|
||||
VIFMacvlanNested = kuryr_kubernetes.cni.binding.nested:MacvlanDriver
|
||||
VIFSRIOV = kuryr_kubernetes.cni.binding.sriov:VIFSRIOVDriver
|
||||
|
||||
kuryr_kubernetes.controller.drivers.pod_project =
|
||||
default = kuryr_kubernetes.controller.drivers.default_project:DefaultPodProjectDriver
|
||||
|
|
|
|||
Loading…
Reference in New Issue