diff --git a/doc/source/installation/index.rst b/doc/source/installation/index.rst index e70ed5598..d17800861 100644 --- a/doc/source/installation/index.rst +++ b/doc/source/installation/index.rst @@ -48,3 +48,4 @@ This section describes how you can install and configure kuryr-kubernetes testing_sriov_functional testing_sctp_services listener_timeouts + multiple_tenants diff --git a/doc/source/installation/multiple_tenants.rst b/doc/source/installation/multiple_tenants.rst new file mode 100644 index 000000000..5f796a32a --- /dev/null +++ b/doc/source/installation/multiple_tenants.rst @@ -0,0 +1,98 @@ +======================== +Multiple tenants support +======================== + + +Annotation project driver +------------------------- + +We introduced an annotation project driver, by the driver you can specify a +openstack project for a k8s namespace, kuryr will take along the project id +when it creates openstack resources (port, subnet, LB, etc.) for the namespace +and the resources (pod, service, etc.) of the namespace. + +Configure to enable the driver in kuryr.conf: + + .. code-block:: ini + + [kubernetes] + pod_project_driver = annotation + service_project_driver = annotation + namespace_project_driver = annotation + network_policy_project_driver = annotation + + +User workflow +~~~~~~~~~~~~~ + +#. Retrieve your own openstack project's id: + + .. code-block:: console + + $ openstack project show test-user + +-------------+----------------------------------+ + | Field | Value | + +-------------+----------------------------------+ + | description | | + | domain_id | default | + | enabled | True | + | id | b5e0a1ae99a34aa0b6a6dad59c95dea7 | + | is_domain | False | + | name | test-user | + | options | {} | + | parent_id | default | + | tags | [] | + +-------------+----------------------------------+ + +#. Create a k8s namespace with the project id + + The manifest file of the namespace: + + .. code-block:: yaml + + apiVersion: v1 + kind: Namespace + metadata: + name: testns + annotations: + openstack.org/kuryr-project: b5e0a1ae99a34aa0b6a6dad59c95dea7 + + Modify the annotation ``openstack.org/kuryr-project``'s value to your own + project id. + +#. Create a pod in the created namespaces: + + .. code-block:: console + + $ kubectl create deployment -n testns --image quay.io/kuryr/demo demo + deployment.apps/demo created + + $ kubectl -n testns get pod -o wide + NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES + demo-6cb99dfd4d-mkjh2 1/1 Running 0 3m15s 10.0.1.76 yjf-dev-kuryr + +#. Retrieve the related openstack resource: + + .. code-block:: console + + $ openstack network list --project b5e0a1ae99a34aa0b6a6dad59c95dea7 + +--------------------------------------+---------------+--------------------------------------+ + | ID | Name | Subnets | + +--------------------------------------+---------------+--------------------------------------+ + | f7e3f025-6d03-40db-b6a8-6671b0874646 | ns/testns-net | d9995087-1363-4671-86da-51b4d17712d8 | + +--------------------------------------+---------------+--------------------------------------+ + + $ openstack subnet list --project b5e0a1ae99a34aa0b6a6dad59c95dea7 + +--------------------------------------+------------------+--------------------------------------+--------------+ + | ID | Name | Network | Subnet | + +--------------------------------------+------------------+--------------------------------------+--------------+ + | d9995087-1363-4671-86da-51b4d17712d8 | ns/testns-subnet | f7e3f025-6d03-40db-b6a8-6671b0874646 | 10.0.1.64/26 | + +--------------------------------------+------------------+--------------------------------------+--------------+ + + $ openstack port list --project b5e0a1ae99a34aa0b6a6dad59c95dea7 + +--------------------------------------+------------------------------+-------------------+--------------------------------------------------------------------------+--------+ + | ID | Name | MAC Address | Fixed IP Addresses | Status | + +--------------------------------------+------------------------------+-------------------+--------------------------------------------------------------------------+--------+ + | 1ce9d0b7-de47-40bb-9bc3-2a8e179681b2 | | fa:16:3e:90:2a:a7 | | DOWN | + | abddd00b-383b-4bf2-9b72-0734739e733d | testns/demo-6cb99dfd4d-mkjh2 | fa:16:3e:a4:c0:f7 | ip_address='10.0.1.76', subnet_id='d9995087-1363-4671-86da-51b4d17712d8' | ACTIVE | + +--------------------------------------+------------------------------+-------------------+--------------------------------------------------------------------------+--------+ diff --git a/kuryr_kubernetes/config.py b/kuryr_kubernetes/config.py index bf4e1cfc8..1318f63a5 100644 --- a/kuryr_kubernetes/config.py +++ b/kuryr_kubernetes/config.py @@ -96,20 +96,20 @@ k8s_opts = [ help=_("The token to talk to the k8s API"), default='/var/run/secrets/kubernetes.io/serviceaccount/token'), cfg.StrOpt('pod_project_driver', - help=_("The driver to determine OpenStack " - "project for pod ports"), + help=_("The driver to determine OpenStack project for pod " + "ports (default or annotation)"), default='default'), cfg.StrOpt('service_project_driver', - help=_("The driver to determine OpenStack " - "project for services"), + help=_("The driver to determine OpenStack project for " + "services (default or annotation)"), default='default'), cfg.StrOpt('namespace_project_driver', - help=_("The driver to determine OpenStack " - "project for namespaces"), + help=_("The driver to determine OpenStack project for " + "namespaces (default or annotation)"), default='default'), cfg.StrOpt('network_policy_project_driver', - help=_("The driver to determine OpenStack " - "project for network policies"), + help=_("The driver to determine OpenStack project for network " + "policies (default or annotation)"), default='default'), cfg.StrOpt('pod_subnets_driver', help=_("The driver to determine Neutron " diff --git a/kuryr_kubernetes/constants.py b/kuryr_kubernetes/constants.py index 4b77eb111..1792f2874 100644 --- a/kuryr_kubernetes/constants.py +++ b/kuryr_kubernetes/constants.py @@ -61,6 +61,7 @@ K8S_ANNOTATION_LBAAS_STATE = K8S_ANNOTATION_PREFIX + '-lbaas-state' K8S_ANNOTATION_NET_CRD = K8S_ANNOTATION_PREFIX + '-net-crd' K8S_ANNOTATION_NETPOLICY_CRD = K8S_ANNOTATION_PREFIX + '-netpolicy-crd' K8S_ANNOTATION_POLICY = K8S_ANNOTATION_PREFIX + '-counter' +K8s_ANNOTATION_PROJECT = K8S_ANNOTATION_PREFIX + '-project' K8S_ANNOTATION_CLIENT_TIMEOUT = K8S_ANNOTATION_PREFIX + '-timeout-client-data' K8S_ANNOTATION_MEMBER_TIMEOUT = K8S_ANNOTATION_PREFIX + '-timeout-member-data' diff --git a/kuryr_kubernetes/controller/drivers/annotation_project.py b/kuryr_kubernetes/controller/drivers/annotation_project.py new file mode 100644 index 000000000..04acc359d --- /dev/null +++ b/kuryr_kubernetes/controller/drivers/annotation_project.py @@ -0,0 +1,69 @@ +# Copyright (c) 2022 Troila +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg +from oslo_log import log as logging + +from kuryr_kubernetes import config +from kuryr_kubernetes import constants +from kuryr_kubernetes.controller.drivers import base +from kuryr_kubernetes.controller.drivers import utils as driver_utils + +LOG = logging.getLogger(__name__) + + +class AnnotationProjectBaseDriver( + base.PodProjectDriver, base.ServiceProjectDriver, + base.NamespaceProjectDriver, base.NetworkPolicyProjectDriver): + """Provides project ID based on resource's annotation.""" + + project_annotation = constants.K8s_ANNOTATION_PROJECT + + def _get_namespace_project(self, namespace): + ns_md = namespace['metadata'] + project = ns_md.get('annotations', {}).get(self.project_annotation) + if not project: + LOG.debug("Namespace %s has no project annotation, try to get " + "project id from the configuration option.", + namespace['metadata']['name']) + project = config.CONF.neutron_defaults.project + if not project: + raise cfg.RequiredOptError('project', + cfg.OptGroup('neutron_defaults')) + return project + + def get_project(self, resource): + res_ns = resource['metadata']['namespace'] + namespace_path = f"{constants.K8S_API_NAMESPACES}/{res_ns}" + namespace = driver_utils.get_k8s_resource(namespace_path) + return self._get_namespace_project(namespace) + + +class AnnotationPodProjectDriver(AnnotationProjectBaseDriver): + pass + + +class AnnotationServiceProjectDriver(AnnotationProjectBaseDriver): + pass + + +class AnnotationNamespaceProjectDriver(AnnotationProjectBaseDriver): + + def get_project(self, namespace): + return self._get_namespace_project(namespace) + + +class AnnotationNetworkPolicyProjectDriver(AnnotationProjectBaseDriver): + pass diff --git a/kuryr_kubernetes/controller/drivers/default_project.py b/kuryr_kubernetes/controller/drivers/default_project.py index 8e3bd1ed2..03ed860e9 100644 --- a/kuryr_kubernetes/controller/drivers/default_project.py +++ b/kuryr_kubernetes/controller/drivers/default_project.py @@ -26,10 +26,6 @@ class DefaultPodProjectDriver(base.PodProjectDriver): project_id = config.CONF.neutron_defaults.project if not project_id: - # NOTE(ivc): this option is only required for - # DefaultPodProjectDriver and its subclasses, but it may be - # optional for other drivers (e.g. when each namespace has own - # project) raise cfg.RequiredOptError('project', cfg.OptGroup('neutron_defaults')) diff --git a/kuryr_kubernetes/controller/handlers/namespace.py b/kuryr_kubernetes/controller/handlers/namespace.py index dd9160eaf..455cda5f7 100644 --- a/kuryr_kubernetes/controller/handlers/namespace.py +++ b/kuryr_kubernetes/controller/handlers/namespace.py @@ -49,7 +49,7 @@ class NamespaceHandler(k8s_base.ResourceEventHandler): return try: - self._add_kuryrnetwork_crd(ns_name, ns_labels) + self._add_kuryrnetwork_crd(namespace, ns_labels) except exceptions.K8sClientException: LOG.exception("Kuryrnetwork CRD creation failed.") raise exceptions.ResourceNotReady(namespace) @@ -104,6 +104,7 @@ class NamespaceHandler(k8s_base.ResourceEventHandler): return kuryrnetwork_crd def _add_kuryrnetwork_crd(self, namespace, ns_labels): + ns_name = namespace['metadata']['name'] project_id = self._drv_project.get_project(namespace) kubernetes = clients.get_kubernetes_client() @@ -111,18 +112,18 @@ class NamespaceHandler(k8s_base.ResourceEventHandler): 'apiVersion': 'openstack.org/v1', 'kind': 'KuryrNetwork', 'metadata': { - 'name': namespace, + 'name': ns_name, 'finalizers': [constants.KURYRNETWORK_FINALIZER], }, 'spec': { - 'nsName': namespace, + 'nsName': ns_name, 'projectId': project_id, 'nsLabels': ns_labels, } } try: kubernetes.post('{}/{}/kuryrnetworks'.format( - constants.K8S_API_CRD_NAMESPACES, namespace), kns_crd) + constants.K8S_API_CRD_NAMESPACES, ns_name), kns_crd) except exceptions.K8sClientException: LOG.exception("Kubernetes Client Exception creating kuryrnetwork " "CRD.") diff --git a/kuryr_kubernetes/tests/unit/controller/drivers/test_annotation_project.py b/kuryr_kubernetes/tests/unit/controller/drivers/test_annotation_project.py new file mode 100644 index 000000000..be6d13815 --- /dev/null +++ b/kuryr_kubernetes/tests/unit/controller/drivers/test_annotation_project.py @@ -0,0 +1,122 @@ +# Copyright (c) 2022 Troila +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from unittest import mock + +from oslo_config import cfg + +from kuryr_kubernetes import constants +from kuryr_kubernetes.controller.drivers import annotation_project +from kuryr_kubernetes.tests import base as test_base + + +class TestAnnotationProjectDriverBase(test_base.TestCase): + + project_id = 'fake_project_id' + + def _get_project_from_namespace(self, resource, driver): + m_get_k8s_res = mock.patch('kuryr_kubernetes.controller.drivers.' + 'utils.get_k8s_resource').start() + m_get_k8s_res.return_value = { + 'metadata': { + 'name': 'fake_namespace', + 'annotations': { + constants.K8s_ANNOTATION_PROJECT: self.project_id}}} + project_id = driver.get_project(resource) + self.assertEqual(self.project_id, project_id) + + def _get_project_from_configure_option(self, resource, driver): + m_cfg = mock.patch('kuryr_kubernetes.config.CONF').start() + m_cfg.neutron_defaults.project = self.project_id + m_get_k8s_res = mock.patch('kuryr_kubernetes.controller.drivers.' + 'utils.get_k8s_resource').start() + m_get_k8s_res.return_value = { + 'metadata': { + 'name': 'fake_namespace', + 'annotations': {}}} + project_id = driver.get_project(resource) + self.assertEqual(self.project_id, project_id) + + def _project_id_not_set(self, resource, driver): + m_cfg = mock.patch('kuryr_kubernetes.config.CONF').start() + m_cfg.neutron_defaults.project = "" + m_get_k8s_res = mock.patch('kuryr_kubernetes.controller.drivers.' + 'utils.get_k8s_resource').start() + m_get_k8s_res.return_value = { + 'metadata': { + 'name': 'fake_namespace', + 'annotations': {}}} + self.assertRaises(cfg.RequiredOptError, driver.get_project, resource) + + +class TestAnnotationPodProjectDriver(TestAnnotationProjectDriverBase): + + pod = {'metadata': {'namespace': 'fake_namespace'}} + + def test_get_project(self): + driver = annotation_project.AnnotationPodProjectDriver() + self._get_project_from_namespace(self.pod, driver) + self._get_project_from_configure_option(self.pod, driver) + self._project_id_not_set(self.pod, driver) + + +class TestAnnotationServiceProjectDriver(TestAnnotationProjectDriverBase): + + service = {'metadata': {'namespace': 'fake_namespace'}} + + def test_get_project(self): + driver = annotation_project.AnnotationPodProjectDriver() + self._get_project_from_namespace(self.service, driver) + self._get_project_from_configure_option(self.service, driver) + self._project_id_not_set(self.service, driver) + + +class TestAnnotationNetworkPolicyProjectDriver( + TestAnnotationProjectDriverBase): + + network_policy = {'metadata': {'namespace': 'fake_namespace'}} + + def test_get_project(self): + driver = annotation_project.AnnotationPodProjectDriver() + self._get_project_from_namespace(self.network_policy, driver) + self._get_project_from_configure_option(self.network_policy, driver) + self._project_id_not_set(self.network_policy, driver) + + +class TestAnnotationNamespaceProjectDriver(test_base.TestCase): + + project_id = 'fake_project_id' + driver = annotation_project.AnnotationNamespaceProjectDriver() + + def test_get_project_from_annotation(self): + namespace = {'metadata': { + 'annotations': { + constants.K8s_ANNOTATION_PROJECT: self.project_id}}} + project_id = self.driver.get_project(namespace) + self.assertEqual(self.project_id, project_id) + + @mock.patch('kuryr_kubernetes.config.CONF') + def test_get_project_from_configure_option(self, m_cfg): + m_cfg.neutron_defaults.project = self.project_id + namespace = {'metadata': {'name': 'fake_namespace'}} + project_id = self.driver.get_project(namespace) + self.assertEqual(self.project_id, project_id) + + @mock.patch('kuryr_kubernetes.config.CONF') + def test_project_not_set(self, m_cfg): + m_cfg.neutron_defaults.project = "" + namespace = {'metadata': {'name': 'fake_namespace'}} + self.assertRaises( + cfg.RequiredOptError, self.driver.get_project, namespace) diff --git a/kuryr_kubernetes/tests/unit/controller/handlers/test_namespace.py b/kuryr_kubernetes/tests/unit/controller/handlers/test_namespace.py index 98984e9df..aba3e77a9 100644 --- a/kuryr_kubernetes/tests/unit/controller/handlers/test_namespace.py +++ b/kuryr_kubernetes/tests/unit/controller/handlers/test_namespace.py @@ -82,7 +82,7 @@ class TestNamespaceHandler(test_base.TestCase): self._get_kns_crd.assert_called_once_with( self._namespace['metadata']['name']) self._add_kuryrnetwork_crd.assert_called_once_with( - self._namespace['metadata']['name'], {}) + self._namespace, {}) def test_on_present_existing(self): net_crd = self._get_crd() @@ -109,7 +109,7 @@ class TestNamespaceHandler(test_base.TestCase): self._get_kns_crd.assert_called_once_with( self._namespace['metadata']['name']) self._add_kuryrnetwork_crd.assert_called_once_with( - self._namespace['metadata']['name'], {}) + self._namespace, {}) @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client') def test_handle_namespace_no_pods(self, m_get_k8s_client): diff --git a/releasenotes/notes/support-specify-project-by-namespace-annotation-18bc6eca729bff5e.yaml b/releasenotes/notes/support-specify-project-by-namespace-annotation-18bc6eca729bff5e.yaml new file mode 100644 index 000000000..6d62acb65 --- /dev/null +++ b/releasenotes/notes/support-specify-project-by-namespace-annotation-18bc6eca729bff5e.yaml @@ -0,0 +1,13 @@ +--- +features: + - | + Introduced a new project driver that is able to specify different project + for each namespace. + + .. code-block:: ini + + [kubernetes] + pod_project_driver = annotation + service_project_driver = annotation + namespace_project_driver = annotation + network_policy_project_driver = annotation diff --git a/setup.cfg b/setup.cfg index e76234fd2..f5f13ad5f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -51,15 +51,19 @@ kuryr_kubernetes.cni.binding = kuryr_kubernetes.controller.drivers.pod_project = default = kuryr_kubernetes.controller.drivers.default_project:DefaultPodProjectDriver + annotation = kuryr_kubernetes.controller.drivers.annotation_project:AnnotationPodProjectDriver kuryr_kubernetes.controller.drivers.service_project = default = kuryr_kubernetes.controller.drivers.default_project:DefaultServiceProjectDriver + annotation = kuryr_kubernetes.controller.drivers.annotation_project:AnnotationServiceProjectDriver kuryr_kubernetes.controller.drivers.namespace_project = default = kuryr_kubernetes.controller.drivers.default_project:DefaultNamespaceProjectDriver + annotation = kuryr_kubernetes.controller.drivers.annotation_project:AnnotationNamespaceProjectDriver kuryr_kubernetes.controller.drivers.network_policy_project = default = kuryr_kubernetes.controller.drivers.default_project:DefaultNetworkPolicyProjectDriver + annotation = kuryr_kubernetes.controller.drivers.annotation_project:AnnotationNetworkPolicyProjectDriver kuryr_kubernetes.controller.drivers.pod_subnets = default = kuryr_kubernetes.controller.drivers.default_subnet:DefaultPodSubnetDriver