diff --git a/doc/source/installation/ports-pool.rst b/doc/source/installation/ports-pool.rst index 64570c6ca..3fa434e01 100644 --- a/doc/source/installation/ports-pool.rst +++ b/doc/source/installation/ports-pool.rst @@ -90,3 +90,38 @@ Note there is a developers tool available at `contrib/pools-management` to create/delete ports in the desired pool(s) as well as to control the amount of existing ports loaded into each pool. For more details on this read the readme file on that folder. + + +Multi pod-vif drivers support with pools +---------------------------------------- + +There is a multi pool driver that supports hybrid environments where some +nodes are Bare Metal while others are running inside VMs, therefore having +different VIF drivers (e.g., neutron and nested-vlan). + +This new multi pool driver is the default pool driver used even if a different +vif_pool_driver is set at the config option. However if the configuration +about the mappings between the different pools and pod vif drivers is not +provided at the pools_vif_drivers config option of vif_pool configuration +section only one pool driver will be loaded -- using the standard +vif_pool_driver and pod_vif_driver config options, i.e., using the one +selected at kuryr.conf options. + +To enable the option of having different pools depending on the node's pod +vif types, you need to state the type of pool that you want for each pod vif +driver, e.g.: + + .. code-block:: ini + + [vif_pool] + pools_vif_drivers=nested:nested-vlan,neutron:neutron-vif + +This will use a pool driver nested to handle the pods whose vif driver is +nested-vlan, and a pool driver neutron to handle the pods whose vif driver is +neutron-vif. When the controller is requesting a vif for a pod in node X, it +will first read the node's annotation about pod_vif driver to use, e.g., +pod_vif: nested-vlan, and then use the corresponding pool driver -- which has +the right pod-vif driver set. + +Note that if no annotation is set on a node, the default pod_vif_driver is +used. diff --git a/kuryr_kubernetes/controller/drivers/base.py b/kuryr_kubernetes/controller/drivers/base.py index 9f698f508..48f290892 100644 --- a/kuryr_kubernetes/controller/drivers/base.py +++ b/kuryr_kubernetes/controller/drivers/base.py @@ -50,20 +50,21 @@ class DriverBase(object): """ @classmethod - def get_instance(cls): + def get_instance(cls, driver_alias=None): """Get an implementing driver instance.""" alias = cls.ALIAS - + driver_name = alias + '_driver' if not driver_alias else driver_alias try: - manager = _DRIVER_MANAGERS[alias] + manager = _DRIVER_MANAGERS[driver_name] except KeyError: - name = config.CONF.kubernetes[alias + '_driver'] + name = (config.CONF.kubernetes[driver_name] if not driver_alias + else driver_alias) manager = stv_driver.DriverManager( namespace="%s.%s" % (_DRIVER_NAMESPACE_BASE, alias), name=name, invoke_on_load=True) - _DRIVER_MANAGERS[alias] = manager + _DRIVER_MANAGERS[driver_name] = manager driver = manager.driver if not isinstance(driver, cls): diff --git a/kuryr_kubernetes/controller/drivers/vif_pool.py b/kuryr_kubernetes/controller/drivers/vif_pool.py index f86a9a80e..797be3afa 100644 --- a/kuryr_kubernetes/controller/drivers/vif_pool.py +++ b/kuryr_kubernetes/controller/drivers/vif_pool.py @@ -22,6 +22,7 @@ import time from kuryr.lib._i18n import _ from kuryr.lib import constants as kl_const from neutronclient.common import exceptions as n_exc +from oslo_cache import core as cache from oslo_config import cfg as oslo_cfg from oslo_log import log as logging from oslo_serialization import jsonutils @@ -34,6 +35,7 @@ from kuryr_kubernetes.controller.drivers import default_subnet from kuryr_kubernetes.controller.managers import pool from kuryr_kubernetes import exceptions from kuryr_kubernetes import os_vif_util as ovu +from kuryr_kubernetes import utils LOG = logging.getLogger(__name__) @@ -53,10 +55,31 @@ vif_pool_driver_opts = [ help=_("Minimun interval (in seconds) " "between pool updates"), default=20), + oslo_cfg.DictOpt('pools_vif_drivers', + help=_("Dict with the pool driver and pod driver to be " + "used. If not set, it will take them from the " + "kubernetes driver options for pool and pod " + "drivers respectively"), + default={}), ] oslo_cfg.CONF.register_opts(vif_pool_driver_opts, "vif_pool") +node_vif_driver_caching_opts = [ + oslo_cfg.BoolOpt('caching', default=True), + oslo_cfg.IntOpt('cache_time', default=3600), +] + +oslo_cfg.CONF.register_opts(node_vif_driver_caching_opts, + "node_driver_caching") + +cache.configure(oslo_cfg.CONF) +node_driver_cache_region = cache.create_region() +MEMOIZE = cache.get_memoization_decorator( + oslo_cfg.CONF, node_driver_cache_region, "node_driver_caching") + +cache.configure_cache_region(oslo_cfg.CONF, node_driver_cache_region) + class NoopVIFPool(base.VIFPoolDriver): """No pool VIFs for Kubernetes Pods""" @@ -618,3 +641,68 @@ class NestedVIFPool(BaseVIFPool): no trunk is specified). """ self._remove_precreated_ports(trunk_ips) + + +class MultiVIFPool(base.VIFPoolDriver): + """Manages pools with different VIF types. + + It manages hybrid deployments containing both Bare Metal and Nested + Kubernetes Pods. To do that it creates a pool per node with a different + pool driver depending on the vif driver that the node is using. + + It assumes a label pod_vif is added to each node to inform about the + driver set for that node. If no label is added, it assumes the default pod + vif: the one specified at kuryr.conf + """ + + def set_vif_driver(self): + self._vif_drvs = {} + pools_vif_drivers = oslo_cfg.CONF.vif_pool.pools_vif_drivers + if not pools_vif_drivers: + pod_vif = oslo_cfg.CONF.kubernetes.pod_vif_driver + drv_vif = base.PodVIFDriver.get_instance() + drv_pool = base.VIFPoolDriver.get_instance() + drv_pool.set_vif_driver(drv_vif) + self._vif_drvs[pod_vif] = drv_pool + return + for pool_driver, pod_driver in pools_vif_drivers.items(): + if not utils.check_suitable_multi_pool_driver_opt(pool_driver, + pod_driver): + LOG.ERROR("The pool and pod driver selected are not " + "compatible. They will be skipped") + raise exceptions.MultiPodDriverPoolConfigurationNotSupported() + drv_vif = base.PodVIFDriver.get_instance(driver_alias=pod_driver) + drv_pool = base.VIFPoolDriver.get_instance( + driver_alias=pool_driver) + drv_pool.set_vif_driver(drv_vif) + self._vif_drvs[pod_driver] = drv_pool + + def request_vif(self, pod, project_id, subnets, security_groups): + pod_vif_type = self._get_pod_vif_type(pod) + return self._vif_drvs[pod_vif_type].request_vif( + pod, project_id, subnets, security_groups) + + def release_vif(self, pod, vif, *argv): + pod_vif_type = self._get_pod_vif_type(pod) + self._vif_drvs[pod_vif_type].release_vif(pod, vif, *argv) + + def activate_vif(self, pod, vif): + pod_vif_type = self._get_pod_vif_type(pod) + self._vif_drvs[pod_vif_type].activate_vif(pod, vif) + + def _get_pod_vif_type(self, pod): + node_name = pod['spec']['nodeName'] + return self._get_node_vif_driver(node_name) + + @MEMOIZE + def _get_node_vif_driver(self, node_name): + kubernetes = clients.get_kubernetes_client() + node_info = kubernetes.get( + constants.K8S_API_BASE + '/nodes/' + node_name) + + labels = node_info['metadata'].get('labels', None) + if labels: + pod_vif = labels.get('pod_vif', + oslo_cfg.CONF.kubernetes.pod_vif_driver) + return pod_vif + return oslo_cfg.CONF.kubernetes.pod_vif_driver diff --git a/kuryr_kubernetes/controller/handlers/vif.py b/kuryr_kubernetes/controller/handlers/vif.py index 83b0b3c39..fdc4680ce 100644 --- a/kuryr_kubernetes/controller/handlers/vif.py +++ b/kuryr_kubernetes/controller/handlers/vif.py @@ -44,13 +44,13 @@ class VIFHandler(k8s_base.ResourceEventHandler): self._drv_project = drivers.PodProjectDriver.get_instance() self._drv_subnets = drivers.PodSubnetsDriver.get_instance() self._drv_sg = drivers.PodSecurityGroupsDriver.get_instance() - self._drv_vif = drivers.PodVIFDriver.get_instance() # REVISIT(ltomasbo): The VIF Handler should not be aware of the pool # directly. Due to the lack of a mechanism to load and set the # VIFHandler driver, for now it is aware of the pool driver, but this # will be reverted as soon as a mechanism is in place. - self._drv_vif_pool = drivers.VIFPoolDriver.get_instance() - self._drv_vif_pool.set_vif_driver(self._drv_vif) + self._drv_vif_pool = drivers.VIFPoolDriver.get_instance( + driver_alias='multi_pool') + self._drv_vif_pool.set_vif_driver() def on_present(self, pod): if self._is_host_network(pod) or not self._is_pending_node(pod): diff --git a/kuryr_kubernetes/exceptions.py b/kuryr_kubernetes/exceptions.py index 8889e76a2..1082a66da 100644 --- a/kuryr_kubernetes/exceptions.py +++ b/kuryr_kubernetes/exceptions.py @@ -51,3 +51,14 @@ class AllowedAddressAlreadyPresent(Exception): 'allowed address pair' on a port is made. Such a condition likely indicates a bad program state or a programming bug. """ + + +class MultiPodDriverPoolConfigurationNotSupported(Exception): + """Exception indicates a wrong configuration of the multi pod driver pool + + This exception is raised when the multi pod driver pool is not properly + configured. This could be due to three different reasons: + 1. One of the pool drivers is not supported + 2. One of the pod drivers is not supported + 3. One of the pod drivers is not supported by its selected pool driver + """ diff --git a/kuryr_kubernetes/opts.py b/kuryr_kubernetes/opts.py index 7b49ab7e8..71602007a 100644 --- a/kuryr_kubernetes/opts.py +++ b/kuryr_kubernetes/opts.py @@ -31,6 +31,7 @@ _kuryr_k8s_opts = [ ('octavia_defaults', config.octavia_defaults), ('cache_defaults', config.cache_defaults), ('subnet_caching', default_subnet.subnet_caching_opts), + ('node_driver_caching', vif_pool.node_vif_driver_caching_opts), ('pool_manager', pool.pool_manager_opts), ('cni_daemon', config.daemon_opts), ('health_server', health.health_server_opts), diff --git a/kuryr_kubernetes/tests/unit/controller/drivers/test_base.py b/kuryr_kubernetes/tests/unit/controller/drivers/test_base.py index dc45e772a..26a1d76ae 100644 --- a/kuryr_kubernetes/tests/unit/controller/drivers/test_base.py +++ b/kuryr_kubernetes/tests/unit/controller/drivers/test_base.py @@ -64,7 +64,7 @@ class TestDriverBase(test_base.TestCase): m_cfg.kubernetes.__getitem__.assert_called_with(cfg_name) m_stv_mgr.assert_called_with(namespace=namespace, name=drv_name, invoke_on_load=True) - m_mgrs.__setitem__.assert_called_once_with(alias, m_mgr) + m_mgrs.__setitem__.assert_called_once_with(cfg_name, m_mgr) @mock.patch.object(d_base, '_DRIVER_MANAGERS') @mock.patch('kuryr_kubernetes.config.CONF') diff --git a/kuryr_kubernetes/utils.py b/kuryr_kubernetes/utils.py index b85b98559..3e0a89182 100644 --- a/kuryr_kubernetes/utils.py +++ b/kuryr_kubernetes/utils.py @@ -15,6 +15,13 @@ from oslo_serialization import jsonutils CONF = cfg.CONF +VALID_MULTI_POD_POOLS_OPTS = {'noop': ['neutron-vif', + 'nested-vlan', + 'nested-macvlan'], + 'neutron': ['neutron-vif'], + 'nested': ['nested-vlan'], + } + def utf8_json_decoder(byte_data): """Deserializes the bytes into UTF-8 encoded JSON. @@ -50,3 +57,7 @@ def get_pod_unique_name(pod): :returns: String with namespace/name of the pod """ return "%(namespace)s/%(name)s" % pod['metadata'] + + +def check_suitable_multi_pool_driver_opt(pool_driver, pod_driver): + return pod_driver in VALID_MULTI_POD_POOLS_OPTS.get(pool_driver, []) diff --git a/releasenotes/notes/multivif-pools-1cec757c77a8c4f8.yaml b/releasenotes/notes/multivif-pools-1cec757c77a8c4f8.yaml new file mode 100644 index 000000000..713a0cb93 --- /dev/null +++ b/releasenotes/notes/multivif-pools-1cec757c77a8c4f8.yaml @@ -0,0 +1,34 @@ +--- +features: + - | + Adds a new multi pool driver to support hybrid environments where some + nodes are Bare Metal while others are running inside VMs, therefore + having different VIF drivers (e.g., neutron and nested-vlan) + + This new multi pool driver is the default pool driver used even if a + different vif_pool_driver is set at the config option. However if the + configuration about the mappings between the different pools and pod vif + drivers is not provided at the pools_vif_drivers config option of vif_pool + configuration section only one pool driver will be loaded -- using the + standard vif_pool_driver and pod_vif_driver config options, i.e., using + the one selected at kuryr.conf options. + + To enable the option of having different pools depending on the node's pod + vif types, you need to state the type of pool that you want for each pod + vif driver, e.g.: + + .. code-block:: ini + + [vif_pool] + pools_vif_drivers=nested:nested-vlan,neutron:neutron-vif + + + This will use a pool driver nested to handle the pods whose vif driver is + nested-vlan, and a pool driver neutron to handle the pods whose vif driver + is neutron-vif. When the controller is requesting a vif for a pod in + node X, it will first read the node's annotation about pod_vif driver to + use, e.g., pod_vif: nested-vlan, and then use the corresponding pool + driver -- which has the right pod-vif driver set. + + Note that if no annotation is set on a node, the default pod_vif_driver + is used. diff --git a/setup.cfg b/setup.cfg index 88ee49eb2..09aa811e5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -73,6 +73,7 @@ kuryr_kubernetes.controller.drivers.vif_pool = noop = kuryr_kubernetes.controller.drivers.vif_pool:NoopVIFPool neutron = kuryr_kubernetes.controller.drivers.vif_pool:NeutronVIFPool nested = kuryr_kubernetes.controller.drivers.vif_pool:NestedVIFPool + multi_pool = kuryr_kubernetes.controller.drivers.vif_pool:MultiVIFPool [files] packages =