From d6dd891befb5d9ee1d9ca942d371326e47c94948 Mon Sep 17 00:00:00 2001 From: Ilya Chukhnakov Date: Mon, 21 Nov 2016 04:18:05 +0300 Subject: [PATCH] Generic VIF controller driver This patch introduces a driver that manages normal Neutron ports to provide VIFs for Kubernetes Pods. Change-Id: Ice32e96e107f7b7331caca3b79c488532710b4a2 Partially-Implements: blueprint kuryr-k8s-integration --- kuryr_kubernetes/config.py | 3 + kuryr_kubernetes/controller/drivers/base.py | 70 ++++++ .../controller/drivers/generic_vif.py | 100 ++++++++ kuryr_kubernetes/exceptions.py | 8 + kuryr_kubernetes/os_vif_util.py | 30 +++ .../controller/drivers/test_generic_vif.py | 232 ++++++++++++++++++ .../tests/unit/test_os_vif_util.py | 70 ++++++ setup.cfg | 3 + 8 files changed, 516 insertions(+) create mode 100644 kuryr_kubernetes/controller/drivers/generic_vif.py create mode 100644 kuryr_kubernetes/tests/unit/controller/drivers/test_generic_vif.py diff --git a/kuryr_kubernetes/config.py b/kuryr_kubernetes/config.py index 37ac55ca5..95ca823e8 100644 --- a/kuryr_kubernetes/config.py +++ b/kuryr_kubernetes/config.py @@ -42,6 +42,9 @@ k8s_opts = [ cfg.StrOpt('pod_security_groups_driver', help=_("The driver to determine Neutron security groups for pods"), default='default'), + cfg.StrOpt('pod_vif_driver', + help=_("The driver that provides VIFs for Kubernetes Pods."), + default='generic'), ] neutron_defaults = [ diff --git a/kuryr_kubernetes/controller/drivers/base.py b/kuryr_kubernetes/controller/drivers/base.py index 2c7e71bdc..980c163cb 100644 --- a/kuryr_kubernetes/controller/drivers/base.py +++ b/kuryr_kubernetes/controller/drivers/base.py @@ -130,3 +130,73 @@ class PodSecurityGroupsDriver(DriverBase): :return: list containing security groups' IDs """ raise NotImplementedError() + + +@six.add_metaclass(abc.ABCMeta) +class PodVIFDriver(DriverBase): + """Manages Neutron ports to provide VIFs for Kubernetes Pods.""" + + ALIAS = 'pod_vif' + + @abc.abstractmethod + def request_vif(self, pod, project_id, subnets, security_groups): + """Links Neutron port to pod and returns it as VIF object. + + Implementing drivers must ensure the Neutron port satisfying the + requested parameters is present and is valid for specified `pod`. It + is up to the implementing drivers to either create new ports on each + request or reuse available ports when possible. + + Implementing drivers may return a VIF object with its `active` field + set to 'False' to indicate that Neutron port requires additional + actions to enable network connectivity after VIF is plugged (e.g. + setting up OpenFlow and/or iptables rules by OpenVSwitch agent). In + that case the Controller will call driver's `activate_vif` method + and the CNI plugin will block until it receives activation + confirmation from the Controller. + + :param pod: dict containing Kubernetes Pod object + :param project_id: OpenStack project ID + :param subnets: dict containing subnet mapping as returned by + `PodSubnetsDriver.get_subnets`. If multiple entries + are present in that mapping, it is guaranteed that + all entries have the same value of `Network.id`. + :param security_groups: list containing security groups' IDs as + returned by + `PodSecurityGroupsDriver.get_security_groups` + :return: VIF object + """ + raise NotImplementedError() + + @abc.abstractmethod + def release_vif(self, pod, vif): + """Unlinks Neutron port corresponding to VIF object from pod. + + Implementing drivers must ensure the port is either deleted or made + available for reuse by `PodVIFDriver.request_vif`. + + :param pod: dict containing Kubernetes Pod object + :param vif: VIF object as returned by `PodVIFDriver.request_vif` + """ + raise NotImplementedError() + + @abc.abstractmethod + def activate_vif(self, pod, vif): + """Updates VIF to become active. + + Implementing drivers should update the specified `vif` object's + `active` field to 'True' but must ensure that the corresponding + Neutron port is fully configured (i.e. the container using the `vif` + can access the requested network resources). + + Implementing drivers may raise `ResourceNotReady` exception to + indicate that port activation should be retried later which will + cause `activate_vif` to be called again with the same arguments. + + This method may be called before, after or while the VIF is being + plugged by the CNI plugin. + + :param pod: dict containing Kubernetes Pod object + :param vif: VIF object as returned by `PodVIFDriver.request_vif` + """ + raise NotImplementedError() diff --git a/kuryr_kubernetes/controller/drivers/generic_vif.py b/kuryr_kubernetes/controller/drivers/generic_vif.py new file mode 100644 index 000000000..7b5d2082c --- /dev/null +++ b/kuryr_kubernetes/controller/drivers/generic_vif.py @@ -0,0 +1,100 @@ +# Copyright (c) 2016 Mirantis, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from kuryr.lib._i18n import _LE +from kuryr.lib import constants as kl_const +from neutronclient.common import exceptions as n_exc +from oslo_log import log as logging + +from kuryr_kubernetes import clients +from kuryr_kubernetes.controller.drivers import base +from kuryr_kubernetes import exceptions as k_exc +from kuryr_kubernetes import os_vif_util as ovu + + +LOG = logging.getLogger(__name__) + + +class GenericPodVIFDriver(base.PodVIFDriver): + """Manages normal Neutron ports to provide VIFs for Kubernetes Pods.""" + + def request_vif(self, pod, project_id, subnets, security_groups): + neutron = clients.get_neutron_client() + + rq = self._get_port_request(pod, project_id, subnets, security_groups) + port = neutron.create_port(rq).get('port') + vif_plugin = self._get_vif_plugin(port) + + return ovu.neutron_to_osvif_vif(vif_plugin, port, subnets) + + def release_vif(self, pod, vif): + neutron = clients.get_neutron_client() + + try: + neutron.delete_port(vif.id) + except n_exc.PortNotFoundClient: + LOG.debug('Unable to release port %s as it no longer exists.', + vif.id) + + def activate_vif(self, pod, vif): + if vif.active: + return + + neutron = clients.get_neutron_client() + port = neutron.show_port(vif.id).get('port') + + if port['status'] != kl_const.PORT_STATUS_ACTIVE: + raise k_exc.ResourceNotReady(vif) + + vif.active = True + + def _get_port_request(self, pod, project_id, subnets, security_groups): + port_req_body = {'project_id': project_id, + 'name': self._get_port_name(pod), + 'network_id': self._get_network_id(subnets), + 'fixed_ips': ovu.osvif_to_neutron_fixed_ips(subnets), + 'device_owner': kl_const.DEVICE_OWNER, + 'device_id': self._get_device_id(pod), + 'admin_state_up': True, + 'binding:host_id': self._get_host_id(pod)} + + if security_groups: + port_req_body['security_groups'] = security_groups + + return {'port': port_req_body} + + def _get_vif_plugin(self, port): + return port.get('binding:vif_type') + + def _get_network_id(self, subnets): + ids = ovu.osvif_to_neutron_network_ids(subnets) + + if len(ids) != 1: + raise k_exc.IntegrityError(_LE( + "Subnet mapping %(subnets)s is not valid: %(num_networks)s " + "unique networks found") % { + 'subnets': subnets, + 'num_networks': len(ids)}) + + return ids[0] + + def _get_port_name(self, pod): + return pod['metadata']['name'] + + def _get_device_id(self, pod): + return pod['metadata']['uid'] + + def _get_host_id(self, pod): + return pod['spec']['nodeName'] diff --git a/kuryr_kubernetes/exceptions.py b/kuryr_kubernetes/exceptions.py index fa2f393f0..47844c5f7 100644 --- a/kuryr_kubernetes/exceptions.py +++ b/kuryr_kubernetes/exceptions.py @@ -13,6 +13,8 @@ # License for the specific language governing permissions and limitations # under the License. +from kuryr.lib._i18n import _LE + class K8sClientException(Exception): pass @@ -22,5 +24,11 @@ class IntegrityError(RuntimeError): pass +class ResourceNotReady(Exception): + def __init__(self, resource): + super(ResourceNotReady, self).__init__(_LE("Resource not ready: %r") + % resource) + + def format_msg(exception): return "%s: %s" % (exception.__class__.__name__, exception) diff --git a/kuryr_kubernetes/os_vif_util.py b/kuryr_kubernetes/os_vif_util.py index 1c8ee678a..e6ccfc82e 100644 --- a/kuryr_kubernetes/os_vif_util.py +++ b/kuryr_kubernetes/os_vif_util.py @@ -13,6 +13,8 @@ # License for the specific language governing permissions and limitations # under the License. +import six + from kuryr.lib._i18n import _LE from kuryr.lib.binding.drivers import utils as kl_utils from kuryr.lib import constants as kl_const @@ -261,3 +263,31 @@ def neutron_to_osvif_vif(vif_plugin, neutron_port, subnets): _VIF_MANAGERS[vif_plugin] = mgr return mgr.driver(vif_plugin, neutron_port, subnets) + + +def osvif_to_neutron_fixed_ips(subnets): + fixed_ips = [] + + for subnet_id, network in six.iteritems(subnets): + ips = [] + if len(network.subnets.objects) > 1: + raise k_exc.IntegrityError(_LE( + "Network object for subnet %(subnet_id)s is invalid, " + "must contain a single subnet, but %(num_subnets)s found") % { + 'subnet_id': subnet_id, + 'num_subnets': len(network.subnets.objects)}) + + for subnet in network.subnets.objects: + if subnet.obj_attr_is_set('ips'): + ips.extend([str(ip.address) for ip in subnet.ips.objects]) + if ips: + fixed_ips.extend([{'subnet_id': subnet_id, 'ip_address': ip} + for ip in ips]) + else: + fixed_ips.append({'subnet_id': subnet_id}) + + return fixed_ips + + +def osvif_to_neutron_network_ids(subnets): + return list(set(net.id for net in six.itervalues(subnets))) diff --git a/kuryr_kubernetes/tests/unit/controller/drivers/test_generic_vif.py b/kuryr_kubernetes/tests/unit/controller/drivers/test_generic_vif.py new file mode 100644 index 000000000..6b3b7f1d2 --- /dev/null +++ b/kuryr_kubernetes/tests/unit/controller/drivers/test_generic_vif.py @@ -0,0 +1,232 @@ +# Copyright (c) 2016 Mirantis, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from kuryr.lib import constants as kl_const +from neutronclient.common import exceptions as n_exc + +from kuryr_kubernetes.controller.drivers import generic_vif +from kuryr_kubernetes import exceptions as k_exc +from kuryr_kubernetes.tests import base as test_base +from kuryr_kubernetes.tests.unit import kuryr_fixtures as k_fix + + +class GenericPodVIFDriver(test_base.TestCase): + + @mock.patch('kuryr_kubernetes.os_vif_util.neutron_to_osvif_vif') + def test_request_vif(self, m_to_vif): + cls = generic_vif.GenericPodVIFDriver + m_driver = mock.Mock(spec=cls) + neutron = self.useFixture(k_fix.MockNeutronClient()).client + + pod = mock.sentinel.pod + project_id = mock.sentinel.project_id + subnets = mock.sentinel.subnets + security_groups = mock.sentinel.security_groups + port = mock.sentinel.port + port_request = mock.sentinel.port_request + vif = mock.sentinel.vif + vif_plugin = mock.sentinel.vif_plugin + + m_to_vif.return_value = vif + m_driver._get_port_request.return_value = port_request + m_driver._get_vif_plugin.return_value = vif_plugin + neutron.create_port.return_value = {'port': port} + + self.assertEqual(vif, cls.request_vif(m_driver, pod, project_id, + subnets, security_groups)) + + m_driver._get_port_request.assert_called_once_with( + pod, project_id, subnets, security_groups) + neutron.create_port.assert_called_once_with(port_request) + m_driver._get_vif_plugin.assert_called_once_with(port) + m_to_vif.assert_called_once_with(vif_plugin, port, subnets) + + def test_release_vif(self): + cls = generic_vif.GenericPodVIFDriver + m_driver = mock.Mock(spec=cls) + neutron = self.useFixture(k_fix.MockNeutronClient()).client + + pod = mock.sentinel.pod + vif = mock.Mock() + + cls.release_vif(m_driver, pod, vif) + + neutron.delete_port.assert_called_once_with(vif.id) + + def test_release_vif_not_found(self): + cls = generic_vif.GenericPodVIFDriver + m_driver = mock.Mock(spec=cls) + neutron = self.useFixture(k_fix.MockNeutronClient()).client + + pod = mock.sentinel.pod + vif = mock.Mock() + neutron.delete_port.side_effect = n_exc.PortNotFoundClient + + cls.release_vif(m_driver, pod, vif) + + neutron.delete_port.assert_called_once_with(vif.id) + + def test_activate_vif(self): + cls = generic_vif.GenericPodVIFDriver + m_driver = mock.Mock(spec=cls) + neutron = self.useFixture(k_fix.MockNeutronClient()).client + + pod = mock.sentinel.pod + vif = mock.Mock() + vif.active = False + port = mock.MagicMock() + + port.__getitem__.return_value = kl_const.PORT_STATUS_ACTIVE + neutron.show_port.return_value = {'port': port} + + cls.activate_vif(m_driver, pod, vif) + + neutron.show_port.assert_called_once_with(vif.id) + self.assertTrue(vif.active) + + def test_activate_vif_active(self): + cls = generic_vif.GenericPodVIFDriver + m_driver = mock.Mock(spec=cls) + neutron = self.useFixture(k_fix.MockNeutronClient()).client + + pod = mock.sentinel.pod + vif = mock.Mock() + vif.active = True + + cls.activate_vif(m_driver, pod, vif) + + neutron.show_port.assert_not_called() + + def test_activate_vif_not_ready(self): + cls = generic_vif.GenericPodVIFDriver + m_driver = mock.Mock(spec=cls) + neutron = self.useFixture(k_fix.MockNeutronClient()).client + + pod = mock.sentinel.pod + vif = mock.Mock() + vif.active = False + port = mock.MagicMock() + + port.__getitem__.return_value = kl_const.PORT_STATUS_DOWN + neutron.show_port.return_value = {'port': port} + + self.assertRaises(k_exc.ResourceNotReady, cls.activate_vif, + m_driver, pod, vif) + + def _test_get_port_request(self, m_to_fips, security_groups): + cls = generic_vif.GenericPodVIFDriver + m_driver = mock.Mock(spec=cls) + + pod = mock.sentinel.pod + project_id = mock.sentinel.project_id + subnets = mock.sentinel.subnets + port_name = mock.sentinel.port_name + network_id = mock.sentinel.project_id + fixed_ips = mock.sentinel.fixed_ips + device_id = mock.sentinel.device_id + host_id = mock.sentinel.host_id + + m_driver._get_port_name.return_value = port_name + m_driver._get_network_id.return_value = network_id + m_to_fips.return_value = fixed_ips + m_driver._get_device_id.return_value = device_id + m_driver._get_host_id.return_value = host_id + + expected = {'port': {'project_id': project_id, + 'name': port_name, + 'network_id': network_id, + 'fixed_ips': fixed_ips, + 'device_owner': kl_const.DEVICE_OWNER, + 'device_id': device_id, + 'admin_state_up': True, + 'binding:host_id': host_id}} + + if security_groups: + expected['port']['security_groups'] = security_groups + + ret = cls._get_port_request(m_driver, pod, project_id, subnets, + security_groups) + + self.assertEqual(expected, ret) + m_driver._get_port_name.assert_called_once_with(pod) + m_driver._get_network_id.assert_called_once_with(subnets) + m_to_fips.assert_called_once_with(subnets) + m_driver._get_device_id.assert_called_once_with(pod) + m_driver._get_host_id.assert_called_once_with(pod) + + @mock.patch('kuryr_kubernetes.os_vif_util.osvif_to_neutron_fixed_ips') + def test_get_port_request(self, m_to_fips): + security_groups = mock.sentinel.security_groups + self._test_get_port_request(m_to_fips, security_groups) + + @mock.patch('kuryr_kubernetes.os_vif_util.osvif_to_neutron_fixed_ips') + def test_get_port_request_no_sg(self, m_to_fips): + security_groups = [] + self._test_get_port_request(m_to_fips, security_groups) + + def test_get_vif_plugin(self): + cls = generic_vif.GenericPodVIFDriver + m_driver = mock.Mock(spec=cls) + vif_plugin = mock.sentinel.vif_plugin + port = {'binding:vif_type': vif_plugin} + + self.assertEqual(vif_plugin, cls._get_vif_plugin(m_driver, port)) + + @mock.patch('kuryr_kubernetes.os_vif_util.osvif_to_neutron_network_ids') + def test_get_network_id(self, m_to_net_ids): + cls = generic_vif.GenericPodVIFDriver + m_driver = mock.Mock(spec=cls) + subnets = mock.sentinel.subnets + network_id = mock.sentinel.network_id + m_to_net_ids.return_value = [network_id] + + self.assertEqual(network_id, cls._get_network_id(m_driver, subnets)) + m_to_net_ids.assert_called_once_with(subnets) + + @mock.patch('kuryr_kubernetes.os_vif_util.osvif_to_neutron_network_ids') + def test_get_network_id_invalid(self, m_to_net_ids): + cls = generic_vif.GenericPodVIFDriver + m_driver = mock.Mock(spec=cls) + subnets = mock.sentinel.subnets + m_to_net_ids.return_value = [] + + self.assertRaises(k_exc.IntegrityError, cls._get_network_id, m_driver, + subnets) + + def test_get_port_name(self): + cls = generic_vif.GenericPodVIFDriver + m_driver = mock.Mock(spec=cls) + pod_name = mock.sentinel.pod_name + pod = {'metadata': {'name': pod_name}} + + self.assertEqual(pod_name, cls._get_port_name(m_driver, pod)) + + def test_get_device_id(self): + cls = generic_vif.GenericPodVIFDriver + m_driver = mock.Mock(spec=cls) + pod_uid = mock.sentinel.pod_uid + pod = {'metadata': {'uid': pod_uid}} + + self.assertEqual(pod_uid, cls._get_device_id(m_driver, pod)) + + def test_get_host_id(self): + cls = generic_vif.GenericPodVIFDriver + m_driver = mock.Mock(spec=cls) + node = mock.sentinel.pod_uid + pod = {'spec': {'nodeName': node}} + + self.assertEqual(node, cls._get_host_id(m_driver, pod)) diff --git a/kuryr_kubernetes/tests/unit/test_os_vif_util.py b/kuryr_kubernetes/tests/unit/test_os_vif_util.py index 21dfc82b2..784ce3da5 100644 --- a/kuryr_kubernetes/tests/unit/test_os_vif_util.py +++ b/kuryr_kubernetes/tests/unit/test_os_vif_util.py @@ -15,7 +15,10 @@ import mock +from os_vif.objects import fixed_ip as osv_fixed_ip +from os_vif.objects import network as osv_network from os_vif.objects import route as osv_route +from os_vif.objects import subnet as osv_subnet from oslo_config import cfg as o_cfg from oslo_utils import uuidutils @@ -352,3 +355,70 @@ class TestOSVIFUtils(test_base.TestCase): self.assertRaises(k_exc.IntegrityError, ovu._make_vif_subnet, subnets, subnet_id) + + def test_osvif_to_neutron_network_ids(self): + id_a = mock.sentinel.id_a + id_b = mock.sentinel.id_b + net1 = mock.Mock() + net1.id = id_a + net2 = mock.Mock() + net2.id = id_b + net3 = mock.Mock() + net3.id = id_a + subnets = {1: net1, 2: net2, 3: net3} + + ret = ovu.osvif_to_neutron_network_ids(subnets) + self.assertEqual(2, len(ret)) + self.assertIn(id_a, ret) + self.assertIn(id_b, ret) + + def test_osvif_to_neutron_fixed_ips(self): + ip11 = '1.1.1.1' + ip12 = '2.2.2.2' + ip3 = '3.3.3.3' + subnet_id_1 = uuidutils.generate_uuid() + subnet_id_2 = uuidutils.generate_uuid() + subnet_id_3 = uuidutils.generate_uuid() + + subnet_1 = osv_subnet.Subnet(ips=osv_fixed_ip.FixedIPList( + objects=[osv_fixed_ip.FixedIP(address=ip11), + osv_fixed_ip.FixedIP(address=ip12)])) + subnet_2 = osv_subnet.Subnet() + subnet_3 = osv_subnet.Subnet(ips=osv_fixed_ip.FixedIPList( + objects=[osv_fixed_ip.FixedIP(address=ip3)])) + + net1 = osv_network.Network(subnets=osv_subnet.SubnetList( + objects=[subnet_1])) + net2 = osv_network.Network(subnets=osv_subnet.SubnetList( + objects=[subnet_2])) + net3 = osv_network.Network(subnets=osv_subnet.SubnetList( + objects=[subnet_3])) + + subnets = {subnet_id_1: net1, subnet_id_2: net2, subnet_id_3: net3} + + expected = [{'subnet_id': subnet_id_1, 'ip_address': ip11}, + {'subnet_id': subnet_id_1, 'ip_address': ip12}, + {'subnet_id': subnet_id_2}, + {'subnet_id': subnet_id_3, 'ip_address': ip3}] + + ret = ovu.osvif_to_neutron_fixed_ips(subnets) + + def _sort_key(e): + return (e.get('subnet_id'), e.get('ip_address')) + + self.assertEqual(sorted(expected, key=_sort_key), + sorted(ret, key=_sort_key)) + + def test_osvif_to_neutron_fixed_ips_invalid(self): + subnet_id = uuidutils.generate_uuid() + + subnet_1 = osv_subnet.Subnet() + subnet_2 = osv_subnet.Subnet() + + net = osv_network.Network(subnets=osv_subnet.SubnetList( + objects=[subnet_1, subnet_2])) + + subnets = {subnet_id: net} + + self.assertRaises(k_exc.IntegrityError, + ovu.osvif_to_neutron_fixed_ips, subnets) diff --git a/setup.cfg b/setup.cfg index 45a8a5b29..63e3cf37c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,6 +38,9 @@ kuryr_kubernetes.controller.drivers.pod_subnets = kuryr_kubernetes.controller.drivers.pod_security_groups = default = kuryr_kubernetes.controller.drivers.default_security_groups:DefaultPodSecurityGroupsDriver +kuryr_kubernetes.controller.drivers.pod_vif = + generic = kuryr_kubernetes.controller.drivers.generic_vif:GenericPodVIFDriver + [files] packages = kuryr_kubernetes