Add SR-IOV pod vif driver

This commit adds SR-IOV driver and new type of VIF to handle SR-IOV requests.
This driver can work as a primary driver and only one driver, but only when kubernetes
will fully support CNI specification.

Now this driver can work in couple with multi vif driver, e.g. NPWGMultiVIFDriver.
(see doc/source/installation/multi_vif_with_npwg_spec.rst)

Also this driver relies on kubernetes SRIOV device plugin.

This commit also adds 'default_physnet_subnets' setting, that should
include a mapping of physnets to neutron subnet IDs, it's necessary to
specify VIF's physnet (subnet id comes from annotation).

To get details how to create pods with sriov interfaces see
doc/source/installation/sriov.rst

Target bp: kuryr-kubernetes-sriov-support
Change-Id: I45c5f1a7fb423ee68731d0ae85f7171e33d0aeeb
Signed-off-by: Danil Golov <d.golov@partner.samsung.com>
Signed-off-by: Vladimir Kuramshin <v.kuramshin@samsung.com>
Signed-off-by: Alexey Perevalov <a.perevalov@samsung.com>

kuryr_kubernetes/config.py

@@ -237,6 +237,15 @@ nested_vif_driver_opts = [
                default=''),
 ]
 
+DEFAULT_PHYSNET_SUBNET_MAPPINGS = {}
+sriov_opts = [
+    cfg.DictOpt('default_physnet_subnets',
+                help=_("A mapping of default subnets for certain physnets "
+                       "in a form of physnet-name:<SUBNET-ID>"),
+                default=DEFAULT_PHYSNET_SUBNET_MAPPINGS),
+]
+
+
 CONF = cfg.CONF
 CONF.register_opts(kuryr_k8s_opts)
 CONF.register_opts(daemon_opts, group='cni_daemon')
@@ -246,6 +255,7 @@ CONF.register_opts(octavia_defaults, group='octavia_defaults')
 CONF.register_opts(cache_defaults, group='cache_defaults')
 CONF.register_opts(ingress, group='ingress')
 CONF.register_opts(nested_vif_driver_opts, group='pod_vif_nested')
+CONF.register_opts(sriov_opts, group='sriov')
 
 CONF.register_opts(lib_config.core_opts)
 CONF.register_opts(lib_config.binding_opts, 'binding')

kuryr_kubernetes/constants.py

@@ -65,3 +65,4 @@ VIF_POOL_SHOW = '/showPool'
 DEFAULT_IFNAME = 'eth0'
 
 ADDITIONAL_IFNAME_PREFIX = 'eth'
+K8S_NPWG_SRIOV_PREFIX = "intel.com/sriov"

kuryr_kubernetes/controller/drivers/sriov.py

@@ -0,0 +1,132 @@
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from kuryr.lib import constants as kl_const
+from oslo_log import log as logging
+
+from kuryr_kubernetes import clients
+from kuryr_kubernetes import config
+from kuryr_kubernetes import constants
+from kuryr_kubernetes.controller.drivers import neutron_vif
+from kuryr_kubernetes.controller.drivers import utils as c_utils
+from kuryr_kubernetes import os_vif_util as ovu
+
+LOG = logging.getLogger(__name__)
+
+
+class SriovVIFDriver(neutron_vif.NeutronPodVIFDriver):
+    """Provides VIFs for SRIOV VF interfaces."""
+
+    ALIAS = 'sriov_pod_vif'
+
+    def __init__(self):
+        self._physnet_mapping = self._get_physnet_mapping()
+
+    def request_vif(self, pod, project_id, subnets, security_groups):
+        amount = self._get_remaining_sriov_vfs(pod)
+        if not amount:
+            LOG.error("SRIOV VIF request failed due to lack of "
+                      "available VFs for the current pod creation")
+            return None
+
+        pod_name = pod['metadata']['name']
+        neutron = clients.get_neutron_client()
+        vif_plugin = 'sriov'
+        subnet_id = next(iter(subnets))
+        physnet = self._get_physnet_for_subnet_id(subnet_id)
+        LOG.debug("Pod {} handling {}".format(pod_name, physnet))
+        rq = self._get_port_request(pod, project_id,
+                                    subnets, security_groups)
+
+        port = neutron.create_port(rq).get('port')
+        vif = ovu.neutron_to_osvif_vif(vif_plugin, port, subnets)
+        vif.physnet = physnet
+
+        LOG.debug("{} vifs are available for the pod {}".format(
+            amount, pod_name))
+
+        self._reduce_remaining_sriov_vfs(pod)
+        return vif
+
+    def activate_vif(self, pod, vif):
+        vif.active = True
+
+    def _get_physnet_mapping(self):
+        physnets = config.CONF.sriov.default_physnet_subnets
+
+        result = {}
+        for name, subnet_id in physnets.items():
+            result[subnet_id] = name
+        return result
+
+    def _get_physnet_for_subnet_id(self, subnet_id):
+        """Returns an appropriate physnet for exact subnet_id from mapping"""
+        try:
+            physnet = self._physnet_mapping[subnet_id]
+        except KeyError as ex:
+            LOG.error("No mapping for subnet {} in {}".format(
+                subnet_id, self._physnet_mapping))
+            raise ex
+        return physnet
+
+    def _get_remaining_sriov_vfs(self, pod):
+        """Returns the number of remaining vfs.
+
+        Returns the number of remaining vfs from the initial number that
+        got allocated for the current pod. This information is stored in
+        pod object.
+        """
+        containers = pod['spec']['containers']
+        total_amount = 0
+        for container in containers:
+            try:
+                requests = container['resources']['requests']
+                amount_value = requests[constants.K8S_NPWG_SRIOV_PREFIX]
+                total_amount += int(amount_value)
+            except KeyError:
+                    continue
+
+        return total_amount
+
+    def _reduce_remaining_sriov_vfs(self, pod):
+        """Reduces number of avaliable vfs for request"""
+        containers = pod['spec']['containers']
+        for container in containers:
+            try:
+                requests = container['resources']['requests']
+                num_of_sriov = int(requests[constants.K8S_NPWG_SRIOV_PREFIX])
+                if num_of_sriov == 0:
+                    continue
+                requests[constants.K8S_NPWG_SRIOV_PREFIX] = (
+                    str(num_of_sriov - 1))
+            except KeyError:
+                    continue
+
+    def _get_port_request(self, pod, project_id, subnets, security_groups):
+        port_req_body = {
+            'project_id': project_id,
+            'name': c_utils.get_port_name(pod),
+            'network_id': c_utils.get_network_id(subnets),
+            'fixed_ips': ovu.osvif_to_neutron_fixed_ips(subnets),
+            'device_owner': kl_const.DEVICE_OWNER + ':sriov',
+            'device_id': c_utils.get_device_id(pod),
+            'admin_state_up': True,
+            'binding:vnic_type': 'direct',
+            'binding:host_id': c_utils.get_host_id(pod),
+        }
+
+        if security_groups:
+            port_req_body['security_groups'] = security_groups
+
+        return {'port': port_req_body}

kuryr_kubernetes/controller/drivers/vif_pool.py

@@ -85,7 +85,8 @@ VIF_TYPE_TO_DRIVER_MAPPING = {
     'VIFOpenVSwitch': 'neutron-vif',
     'VIFBridge': 'neutron-vif',
     'VIFVlanNested': 'nested-vlan',
-    'VIFMacvlanNested': 'nested-macvlan'
+    'VIFMacvlanNested': 'nested-macvlan',
+    'VIFSriov': 'sriov'
 }
 
 

kuryr_kubernetes/objects/vif.py

@@ -68,3 +68,15 @@ class VIFMacvlanNested(obj_osvif.VIFBase):
         # Name of the device to create
         'vif_name': obj_fields.StringField(),
     }
+
+
+@obj_base.VersionedObjectRegistry.register
+class VIFSriov(obj_osvif.VIFDirect):
+    # This is OVO based SRIOV vif.
+
+    VERSION = '1.0'
+
+    fields = {
+        # physnet of the VIF
+        'physnet': obj_fields.StringField(),
+    }

kuryr_kubernetes/opts.py

@@ -40,6 +40,7 @@ _kuryr_k8s_opts = [
     ('namespace_subnet', namespace_subnet.namespace_subnet_driver_opts),
     ('namespace_sg', namespace_security_groups.namespace_sg_driver_opts),
     ('ingress', config.ingress),
+    ('sriov', config.sriov_opts),
 ]
 
 

kuryr_kubernetes/os_vif_plug_noop.py

@@ -38,3 +38,23 @@ class NoOpPlugin(PluginBase):
 
     def unplug(self, vif, instance_info):
         pass
+
+
+class SriovPlugin(PluginBase):
+    """Sriov Plugin to be used with sriov VIFS"""
+
+    def describe(self):
+        return objects.host_info.HostPluginInfo(
+            plugin_name='sriov',
+            vif_info=[
+                objects.host_info.HostVIFInfo(
+                    vif_object_name=objects.vif.VIFDirect.__name__,
+                    min_version="1.0",
+                    max_version="1.0"),
+            ])
+
+    def plug(self, vif, instance_info):
+        pass
+
+    def unplug(self, vif, instance_info):
+        pass

kuryr_kubernetes/os_vif_util.py

@@ -52,6 +52,11 @@ def neutron_to_osvif_network(neutron_network):
     if neutron_network.get('mtu') is not None:
         obj.mtu = neutron_network['mtu']
 
+    # Vlan information will be used later in Sriov binding driver
+    if neutron_network.get('provider:network_type') == 'vlan':
+        obj.should_provide_vlan = True
+        obj.vlan = neutron_network['provider:segmentation_id']
+
     return obj
 
 
@@ -295,6 +300,35 @@ def neutron_to_osvif_vif_nested_macvlan(neutron_port, subnets):
         vif_name=_get_vif_name(neutron_port))
 
 
+def neutron_to_osvif_vif_sriov(vif_plugin, neutron_port, subnets):
+    """Converts Neutron port to VIF object for SRIOV containers.
+
+    :param vif_plugin: name of the os-vif plugin to use (i.e. 'noop')
+    :param neutron_port: dict containing port information as returned by
+                         neutron client's 'show_port'
+    :param subnets: subnet mapping as returned by PodSubnetsDriver.get_subnets
+    :return: osv_vif VIFSriov object
+    """
+
+    details = neutron_port.get('binding:vif_details', {})
+    network = _make_vif_network(neutron_port, subnets)
+    vlan_name = network.vlan if network.should_provide_vlan else ''
+    vif = k_vif.VIFSriov(
+        id=neutron_port['id'],
+        address=neutron_port['mac_address'],
+        network=network,
+        has_traffic_filtering=details.get('port_filter', False),
+        preserve_on_delete=False,
+        active=_is_port_active(neutron_port),
+        plugin=vif_plugin,
+        mode='passthrough',
+        vlan_name=vlan_name,
+        vif_name=_get_vif_name(neutron_port),
+    )
+
+    return vif
+
+
 def neutron_to_osvif_vif(vif_translator, neutron_port, subnets):
     """Converts Neutron port to os-vif VIF object.
 

kuryr_kubernetes/tests/unit/controller/drivers/test_sriov.py

@@ -0,0 +1,159 @@
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+
+from kuryr_kubernetes.controller.drivers import sriov as sriov_drivers
+from kuryr_kubernetes.tests import base as test_base
+from kuryr_kubernetes.tests.unit import kuryr_fixtures as k_fix
+
+from kuryr_kubernetes import constants as k_const
+from kuryr_kubernetes import os_vif_util as ovu
+from kuryr_kubernetes import utils
+
+from oslo_config import cfg as oslo_cfg
+from oslo_utils import uuidutils
+
+
+class TestSriovVIFDriver(test_base.TestCase):
+
+    def setUp(self):
+        super(TestSriovVIFDriver, self).setUp()
+        self._pod = {
+            'metadata': {
+                'resourceVersion': mock.sentinel.pod_version,
+                'selfLink': mock.sentinel.pod_link,
+                'name': 'podname'},
+            'status': {'phase': k_const.K8S_POD_STATUS_PENDING},
+            'spec': {
+                'hostNetwork': False,
+                'nodeName': 'hostname',
+                'containers': [{
+                    'resources': {
+                        'requests': {
+                            k_const.K8S_NPWG_SRIOV_PREFIX: "2"
+                            }
+                        }
+                    }]
+                }
+            }
+
+    def test_activate_vif(self):
+        cls = sriov_drivers.SriovVIFDriver
+        m_driver = mock.Mock(spec=cls)
+
+        pod = mock.sentinel.pod
+        vif = mock.Mock()
+        vif.active = False
+
+        cls.activate_vif(m_driver, pod, vif)
+        self.assertEqual(True, vif.active)
+
+    @mock.patch('kuryr_kubernetes.os_vif_util.osvif_to_neutron_fixed_ips')
+    @mock.patch.object(ovu, 'neutron_to_osvif_vif')
+    def test_request_vif(self, m_to_vif, m_to_fips):
+        cls = sriov_drivers.SriovVIFDriver
+        m_driver = mock.Mock(spec=cls)
+
+        neutron = self.useFixture(k_fix.MockNeutronClient()).client
+        project_id = mock.sentinel.project_id
+        fixed_ips = mock.sentinel.fixed_ips
+        m_to_fips.return_value = fixed_ips
+        network = mock.sentinel.Network
+        subnet_id = uuidutils.generate_uuid()
+        subnets = {subnet_id: network}
+        security_groups = mock.sentinel.security_groups
+        port_fixed_ips = mock.sentinel.port_fixed_ips
+        port_id = mock.sentinel.port_id
+        port = {
+            'fixed_ips': port_fixed_ips,
+            'id': port_id
+        }
+        port_request = mock.sentinel.port_request
+        m_driver._get_port_request.return_value = port_request
+        vif = mock.sentinel.vif
+        m_to_vif.return_value = vif
+        neutron.create_port.return_value = {'port': port}
+        utils.get_subnet.return_value = subnets
+
+        self.assertEqual(vif, cls.request_vif(m_driver, self._pod, project_id,
+                                              subnets, security_groups))
+
+        neutron.create_port.assert_called_once_with(port_request)
+
+    @mock.patch('kuryr_kubernetes.os_vif_util.osvif_to_neutron_fixed_ips')
+    @mock.patch.object(ovu, 'neutron_to_osvif_vif')
+    def test_request_vif_not_enough_vfs(self, m_to_vif, m_to_fips):
+        cls = sriov_drivers.SriovVIFDriver
+        m_driver = mock.Mock(spec=cls)
+
+        m_driver._get_remaining_sriov_vfs.return_value = 0
+        neutron = self.useFixture(k_fix.MockNeutronClient()).client
+        project_id = mock.sentinel.project_id
+        network = mock.sentinel.Network
+        subnet_id = uuidutils.generate_uuid()
+        subnets = {subnet_id: network}
+        security_groups = mock.sentinel.security_groups
+
+        self.assertIsNone(cls.request_vif(m_driver, self._pod, project_id,
+                                          subnets, security_groups))
+
+        neutron.create_port.assert_not_called()
+
+    def test_get_sriov_num_vf(self):
+        cls = sriov_drivers.SriovVIFDriver
+        m_driver = mock.Mock(spec=cls)
+
+        amount = cls._get_remaining_sriov_vfs(m_driver, self._pod)
+        self.assertEqual(amount, 2)
+
+    def test_reduce_remaining_sriov_vfs(self):
+        cls = sriov_drivers.SriovVIFDriver
+        m_driver = mock.Mock(spec=cls)
+
+        cls._reduce_remaining_sriov_vfs(m_driver, self._pod)
+        amount = cls._get_remaining_sriov_vfs(m_driver, self._pod)
+        self.assertEqual(amount, 1)
+
+    def test_get_physnet_mapping(self):
+        cls = sriov_drivers.SriovVIFDriver
+        m_driver = mock.Mock(spec=cls)
+
+        subnet_id = uuidutils.generate_uuid()
+        oslo_cfg.CONF.set_override('default_physnet_subnets',
+                                   'physnet10_4:'+str(subnet_id),
+                                   group='sriov')
+
+        mapping = cls._get_physnet_mapping(m_driver)
+        self.assertEqual(mapping, {subnet_id: 'physnet10_4'})
+
+    def test_get_physnet_for_subnet_id(self):
+        cls = sriov_drivers.SriovVIFDriver
+        m_driver = mock.Mock(spec=cls)
+
+        subnet_id = uuidutils.generate_uuid()
+        m_driver._physnet_mapping = {subnet_id: 'physnet10_4'}
+
+        physnet = cls._get_physnet_for_subnet_id(m_driver, subnet_id)
+        self.assertEqual(physnet, 'physnet10_4')
+
+    def test_get_physnet_for_subnet_id_error(self):
+        cls = sriov_drivers.SriovVIFDriver
+        m_driver = mock.Mock(spec=cls)
+
+        subnet_id = uuidutils.generate_uuid()
+        m_driver._physnet_mapping = {}
+
+        self.assertRaises(KeyError, cls._get_physnet_for_subnet_id,
+                          m_driver, subnet_id)

kuryr_kubernetes/utils.py

@@ -33,7 +33,8 @@ LOG = log.getLogger(__name__)
 
 VALID_MULTI_POD_POOLS_OPTS = {'noop': ['neutron-vif',
                                        'nested-vlan',
-                                       'nested-macvlan'],
+                                       'nested-macvlan',
+                                       'sriov'],
                               'neutron': ['neutron-vif'],
                               'nested': ['nested-vlan'],
                               }

setup.cfg

@@ -25,6 +25,7 @@ oslo.config.opts =
 
 os_vif =
     noop = kuryr_kubernetes.os_vif_plug_noop:NoOpPlugin
+    sriov = kuryr_kubernetes.os_vif_plug_noop:SriovPlugin
 
 console_scripts =
     kuryr-k8s-controller = kuryr_kubernetes.cmd.eventlet.controller:start
@@ -33,6 +34,7 @@ console_scripts =
 
 kuryr_kubernetes.vif_translators =
     ovs = kuryr_kubernetes.os_vif_util:neutron_to_osvif_vif_ovs
+    sriov = kuryr_kubernetes.os_vif_util:neutron_to_osvif_vif_sriov
 
 kuryr_kubernetes.cni.binding =
     VIFBridge = kuryr_kubernetes.cni.binding.bridge:BridgeDriver
@@ -74,6 +76,7 @@ kuryr_kubernetes.controller.drivers.pod_vif =
     neutron-vif = kuryr_kubernetes.controller.drivers.neutron_vif:NeutronPodVIFDriver
     nested-vlan = kuryr_kubernetes.controller.drivers.nested_vlan_vif:NestedVlanPodVIFDriver
     nested-macvlan = kuryr_kubernetes.controller.drivers.nested_macvlan_vif:NestedMacvlanPodVIFDriver
+    sriov = kuryr_kubernetes.controller.drivers.sriov:SriovVIFDriver
 
 kuryr_kubernetes.controller.drivers.endpoints_lbaas =
     lbaasv2 = kuryr_kubernetes.controller.drivers.lbaasv2:LBaaSv2Driver