Merge "Support specify project id by annotation"

doc/source/installation/index.rst

@@ -48,3 +48,4 @@ This section describes how you can install and configure kuryr-kubernetes
    testing_sriov_functional
    testing_sctp_services
    listener_timeouts
+   multiple_tenants

doc/source/installation/multiple_tenants.rst

@@ -0,0 +1,98 @@
+========================
+Multiple tenants support
+========================
+
+
+Annotation project driver
+-------------------------
+
+We introduced an annotation project driver, by the driver you can specify a
+openstack project for a k8s namespace, kuryr will take along the project id
+when it creates openstack resources (port, subnet, LB, etc.) for the namespace
+and the resources (pod, service, etc.) of the namespace.
+
+Configure to enable the driver in kuryr.conf:
+
+    .. code-block:: ini
+
+      [kubernetes]
+      pod_project_driver = annotation
+      service_project_driver = annotation
+      namespace_project_driver = annotation
+      network_policy_project_driver = annotation
+
+
+User workflow
+~~~~~~~~~~~~~
+
+#. Retrieve your own openstack project's id:
+
+    .. code-block:: console
+
+      $ openstack project show test-user
+      +-------------+----------------------------------+
+      | Field       | Value                            |
+      +-------------+----------------------------------+
+      | description |                                  |
+      | domain_id   | default                          |
+      | enabled     | True                             |
+      | id          | b5e0a1ae99a34aa0b6a6dad59c95dea7 |
+      | is_domain   | False                            |
+      | name        | test-user                        |
+      | options     | {}                               |
+      | parent_id   | default                          |
+      | tags        | []                               |
+      +-------------+----------------------------------+
+
+#. Create a k8s namespace with the project id
+
+    The manifest file of the namespace:
+
+    .. code-block:: yaml
+
+      apiVersion: v1
+      kind: Namespace
+      metadata:
+        name: testns
+        annotations:
+          openstack.org/kuryr-project: b5e0a1ae99a34aa0b6a6dad59c95dea7
+
+    Modify the annotation ``openstack.org/kuryr-project``'s value to your own
+    project id.
+
+#. Create a pod in the created namespaces:
+
+    .. code-block:: console
+
+      $ kubectl create deployment -n testns --image quay.io/kuryr/demo demo
+      deployment.apps/demo created
+
+      $ kubectl -n testns get pod -o wide
+      NAME                    READY   STATUS    RESTARTS   AGE     IP          NODE            NOMINATED NODE   READINESS GATES
+      demo-6cb99dfd4d-mkjh2   1/1     Running   0          3m15s   10.0.1.76   yjf-dev-kuryr   <none>           <none>
+
+#. Retrieve the related openstack resource:
+
+    .. code-block:: console
+
+      $ openstack network list --project b5e0a1ae99a34aa0b6a6dad59c95dea7
+      +--------------------------------------+---------------+--------------------------------------+
+      | ID                                   | Name          | Subnets                              |
+      +--------------------------------------+---------------+--------------------------------------+
+      | f7e3f025-6d03-40db-b6a8-6671b0874646 | ns/testns-net | d9995087-1363-4671-86da-51b4d17712d8 |
+      +--------------------------------------+---------------+--------------------------------------+
+
+      $ openstack subnet list --project b5e0a1ae99a34aa0b6a6dad59c95dea7
+      +--------------------------------------+------------------+--------------------------------------+--------------+
+      | ID                                   | Name             | Network                              | Subnet       |
+      +--------------------------------------+------------------+--------------------------------------+--------------+
+      | d9995087-1363-4671-86da-51b4d17712d8 | ns/testns-subnet | f7e3f025-6d03-40db-b6a8-6671b0874646 | 10.0.1.64/26 |
+      +--------------------------------------+------------------+--------------------------------------+--------------+
+
+      $ openstack port list --project b5e0a1ae99a34aa0b6a6dad59c95dea7
+      +--------------------------------------+------------------------------+-------------------+--------------------------------------------------------------------------+--------+
+      | ID                                   | Name                         | MAC Address       | Fixed IP Addresses                                                       | Status |
+      +--------------------------------------+------------------------------+-------------------+--------------------------------------------------------------------------+--------+
+      | 1ce9d0b7-de47-40bb-9bc3-2a8e179681b2 |                              | fa:16:3e:90:2a:a7 |                                                                          | DOWN   |
+      | abddd00b-383b-4bf2-9b72-0734739e733d | testns/demo-6cb99dfd4d-mkjh2 | fa:16:3e:a4:c0:f7 | ip_address='10.0.1.76', subnet_id='d9995087-1363-4671-86da-51b4d17712d8' | ACTIVE |
+      +--------------------------------------+------------------------------+-------------------+--------------------------------------------------------------------------+--------+

kuryr_kubernetes/config.py

@@ -96,20 +96,20 @@ k8s_opts = [
                help=_("The token to talk to the k8s API"),
                default='/var/run/secrets/kubernetes.io/serviceaccount/token'),
     cfg.StrOpt('pod_project_driver',
-               help=_("The driver to determine OpenStack "
+               help=_("The driver to determine OpenStack project for pod "
-                      "project for pod ports"),
+                      "ports (default or annotation)"),
                default='default'),
     cfg.StrOpt('service_project_driver',
-               help=_("The driver to determine OpenStack "
+               help=_("The driver to determine OpenStack project for "
-                      "project for services"),
+                      "services (default or annotation)"),
                default='default'),
     cfg.StrOpt('namespace_project_driver',
-               help=_("The driver to determine OpenStack "
+               help=_("The driver to determine OpenStack project for "
-                      "project for namespaces"),
+                      "namespaces (default or annotation)"),
                default='default'),
     cfg.StrOpt('network_policy_project_driver',
-               help=_("The driver to determine OpenStack "
+               help=_("The driver to determine OpenStack project for network "
-                      "project for network policies"),
+                      "policies (default or annotation)"),
                default='default'),
     cfg.StrOpt('pod_subnets_driver',
                help=_("The driver to determine Neutron "

kuryr_kubernetes/constants.py

@@ -61,6 +61,7 @@ K8S_ANNOTATION_LBAAS_STATE = K8S_ANNOTATION_PREFIX + '-lbaas-state'
 K8S_ANNOTATION_NET_CRD = K8S_ANNOTATION_PREFIX + '-net-crd'
 K8S_ANNOTATION_NETPOLICY_CRD = K8S_ANNOTATION_PREFIX + '-netpolicy-crd'
 K8S_ANNOTATION_POLICY = K8S_ANNOTATION_PREFIX + '-counter'
+K8s_ANNOTATION_PROJECT = K8S_ANNOTATION_PREFIX + '-project'
 
 K8S_ANNOTATION_CLIENT_TIMEOUT = K8S_ANNOTATION_PREFIX + '-timeout-client-data'
 K8S_ANNOTATION_MEMBER_TIMEOUT = K8S_ANNOTATION_PREFIX + '-timeout-member-data'

kuryr_kubernetes/controller/drivers/annotation_project.py

@@ -0,0 +1,69 @@
+# Copyright (c) 2022 Troila
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_config import cfg
+from oslo_log import log as logging
+
+from kuryr_kubernetes import config
+from kuryr_kubernetes import constants
+from kuryr_kubernetes.controller.drivers import base
+from kuryr_kubernetes.controller.drivers import utils as driver_utils
+
+LOG = logging.getLogger(__name__)
+
+
+class AnnotationProjectBaseDriver(
+        base.PodProjectDriver, base.ServiceProjectDriver,
+        base.NamespaceProjectDriver, base.NetworkPolicyProjectDriver):
+    """Provides project ID based on resource's annotation."""
+
+    project_annotation = constants.K8s_ANNOTATION_PROJECT
+
+    def _get_namespace_project(self, namespace):
+        ns_md = namespace['metadata']
+        project = ns_md.get('annotations', {}).get(self.project_annotation)
+        if not project:
+            LOG.debug("Namespace %s has no project annotation, try to get "
+                      "project id from the configuration option.",
+                      namespace['metadata']['name'])
+            project = config.CONF.neutron_defaults.project
+        if not project:
+            raise cfg.RequiredOptError('project',
+                                       cfg.OptGroup('neutron_defaults'))
+        return project
+
+    def get_project(self, resource):
+        res_ns = resource['metadata']['namespace']
+        namespace_path = f"{constants.K8S_API_NAMESPACES}/{res_ns}"
+        namespace = driver_utils.get_k8s_resource(namespace_path)
+        return self._get_namespace_project(namespace)
+
+
+class AnnotationPodProjectDriver(AnnotationProjectBaseDriver):
+    pass
+
+
+class AnnotationServiceProjectDriver(AnnotationProjectBaseDriver):
+    pass
+
+
+class AnnotationNamespaceProjectDriver(AnnotationProjectBaseDriver):
+
+    def get_project(self, namespace):
+        return self._get_namespace_project(namespace)
+
+
+class AnnotationNetworkPolicyProjectDriver(AnnotationProjectBaseDriver):
+    pass

kuryr_kubernetes/controller/drivers/default_project.py

@@ -26,10 +26,6 @@ class DefaultPodProjectDriver(base.PodProjectDriver):
         project_id = config.CONF.neutron_defaults.project
 
         if not project_id:
-            # NOTE(ivc): this option is only required for
-            # DefaultPodProjectDriver and its subclasses, but it may be
-            # optional for other drivers (e.g. when each namespace has own
-            # project)
             raise cfg.RequiredOptError('project',
                                        cfg.OptGroup('neutron_defaults'))
 

kuryr_kubernetes/controller/handlers/namespace.py

@@ -49,7 +49,7 @@ class NamespaceHandler(k8s_base.ResourceEventHandler):
             return
 
         try:
-            self._add_kuryrnetwork_crd(ns_name, ns_labels)
+            self._add_kuryrnetwork_crd(namespace, ns_labels)
         except exceptions.K8sClientException:
             LOG.exception("Kuryrnetwork CRD creation failed.")
             raise exceptions.ResourceNotReady(namespace)
@@ -104,6 +104,7 @@ class NamespaceHandler(k8s_base.ResourceEventHandler):
         return kuryrnetwork_crd
 
     def _add_kuryrnetwork_crd(self, namespace, ns_labels):
+        ns_name = namespace['metadata']['name']
         project_id = self._drv_project.get_project(namespace)
         kubernetes = clients.get_kubernetes_client()
 
@@ -111,18 +112,18 @@ class NamespaceHandler(k8s_base.ResourceEventHandler):
             'apiVersion': 'openstack.org/v1',
             'kind': 'KuryrNetwork',
             'metadata': {
-                'name': namespace,
+                'name': ns_name,
                 'finalizers': [constants.KURYRNETWORK_FINALIZER],
             },
             'spec': {
-                'nsName': namespace,
+                'nsName': ns_name,
                 'projectId': project_id,
                 'nsLabels': ns_labels,
             }
         }
         try:
             kubernetes.post('{}/{}/kuryrnetworks'.format(
-                constants.K8S_API_CRD_NAMESPACES, namespace), kns_crd)
+                constants.K8S_API_CRD_NAMESPACES, ns_name), kns_crd)
         except exceptions.K8sClientException:
             LOG.exception("Kubernetes Client Exception creating kuryrnetwork "
                           "CRD.")

kuryr_kubernetes/tests/unit/controller/drivers/test_annotation_project.py

@@ -0,0 +1,122 @@
+# Copyright (c) 2022 Troila
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from unittest import mock
+
+from oslo_config import cfg
+
+from kuryr_kubernetes import constants
+from kuryr_kubernetes.controller.drivers import annotation_project
+from kuryr_kubernetes.tests import base as test_base
+
+
+class TestAnnotationProjectDriverBase(test_base.TestCase):
+
+    project_id = 'fake_project_id'
+
+    def _get_project_from_namespace(self, resource, driver):
+        m_get_k8s_res = mock.patch('kuryr_kubernetes.controller.drivers.'
+                                   'utils.get_k8s_resource').start()
+        m_get_k8s_res.return_value = {
+            'metadata': {
+                'name': 'fake_namespace',
+                'annotations': {
+                    constants.K8s_ANNOTATION_PROJECT: self.project_id}}}
+        project_id = driver.get_project(resource)
+        self.assertEqual(self.project_id, project_id)
+
+    def _get_project_from_configure_option(self, resource, driver):
+        m_cfg = mock.patch('kuryr_kubernetes.config.CONF').start()
+        m_cfg.neutron_defaults.project = self.project_id
+        m_get_k8s_res = mock.patch('kuryr_kubernetes.controller.drivers.'
+                                   'utils.get_k8s_resource').start()
+        m_get_k8s_res.return_value = {
+            'metadata': {
+                'name': 'fake_namespace',
+                'annotations': {}}}
+        project_id = driver.get_project(resource)
+        self.assertEqual(self.project_id, project_id)
+
+    def _project_id_not_set(self, resource, driver):
+        m_cfg = mock.patch('kuryr_kubernetes.config.CONF').start()
+        m_cfg.neutron_defaults.project = ""
+        m_get_k8s_res = mock.patch('kuryr_kubernetes.controller.drivers.'
+                                   'utils.get_k8s_resource').start()
+        m_get_k8s_res.return_value = {
+            'metadata': {
+                'name': 'fake_namespace',
+                'annotations': {}}}
+        self.assertRaises(cfg.RequiredOptError, driver.get_project, resource)
+
+
+class TestAnnotationPodProjectDriver(TestAnnotationProjectDriverBase):
+
+    pod = {'metadata': {'namespace': 'fake_namespace'}}
+
+    def test_get_project(self):
+        driver = annotation_project.AnnotationPodProjectDriver()
+        self._get_project_from_namespace(self.pod, driver)
+        self._get_project_from_configure_option(self.pod, driver)
+        self._project_id_not_set(self.pod, driver)
+
+
+class TestAnnotationServiceProjectDriver(TestAnnotationProjectDriverBase):
+
+    service = {'metadata': {'namespace': 'fake_namespace'}}
+
+    def test_get_project(self):
+        driver = annotation_project.AnnotationPodProjectDriver()
+        self._get_project_from_namespace(self.service, driver)
+        self._get_project_from_configure_option(self.service, driver)
+        self._project_id_not_set(self.service, driver)
+
+
+class TestAnnotationNetworkPolicyProjectDriver(
+        TestAnnotationProjectDriverBase):
+
+    network_policy = {'metadata': {'namespace': 'fake_namespace'}}
+
+    def test_get_project(self):
+        driver = annotation_project.AnnotationPodProjectDriver()
+        self._get_project_from_namespace(self.network_policy, driver)
+        self._get_project_from_configure_option(self.network_policy, driver)
+        self._project_id_not_set(self.network_policy, driver)
+
+
+class TestAnnotationNamespaceProjectDriver(test_base.TestCase):
+
+    project_id = 'fake_project_id'
+    driver = annotation_project.AnnotationNamespaceProjectDriver()
+
+    def test_get_project_from_annotation(self):
+        namespace = {'metadata': {
+            'annotations': {
+                constants.K8s_ANNOTATION_PROJECT: self.project_id}}}
+        project_id = self.driver.get_project(namespace)
+        self.assertEqual(self.project_id, project_id)
+
+    @mock.patch('kuryr_kubernetes.config.CONF')
+    def test_get_project_from_configure_option(self, m_cfg):
+        m_cfg.neutron_defaults.project = self.project_id
+        namespace = {'metadata': {'name': 'fake_namespace'}}
+        project_id = self.driver.get_project(namespace)
+        self.assertEqual(self.project_id, project_id)
+
+    @mock.patch('kuryr_kubernetes.config.CONF')
+    def test_project_not_set(self, m_cfg):
+        m_cfg.neutron_defaults.project = ""
+        namespace = {'metadata': {'name': 'fake_namespace'}}
+        self.assertRaises(
+            cfg.RequiredOptError, self.driver.get_project, namespace)

kuryr_kubernetes/tests/unit/controller/handlers/test_namespace.py

@@ -82,7 +82,7 @@ class TestNamespaceHandler(test_base.TestCase):
         self._get_kns_crd.assert_called_once_with(
             self._namespace['metadata']['name'])
         self._add_kuryrnetwork_crd.assert_called_once_with(
-            self._namespace['metadata']['name'], {})
+            self._namespace, {})
 
     def test_on_present_existing(self):
         net_crd = self._get_crd()
@@ -109,7 +109,7 @@ class TestNamespaceHandler(test_base.TestCase):
         self._get_kns_crd.assert_called_once_with(
             self._namespace['metadata']['name'])
         self._add_kuryrnetwork_crd.assert_called_once_with(
-            self._namespace['metadata']['name'], {})
+            self._namespace, {})
 
     @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client')
     def test_handle_namespace_no_pods(self, m_get_k8s_client):

releasenotes/notes/support-specify-project-by-namespace-annotation-18bc6eca729bff5e.yaml

@@ -0,0 +1,13 @@
+---
+features:
+  - |
+    Introduced a new project driver that is able to specify different project
+    for each namespace.
+
+    .. code-block:: ini
+
+      [kubernetes]
+      pod_project_driver = annotation
+      service_project_driver = annotation
+      namespace_project_driver = annotation
+      network_policy_project_driver = annotation

setup.cfg

@@ -51,15 +51,19 @@ kuryr_kubernetes.cni.binding =
 
 kuryr_kubernetes.controller.drivers.pod_project =
     default = kuryr_kubernetes.controller.drivers.default_project:DefaultPodProjectDriver
+    annotation = kuryr_kubernetes.controller.drivers.annotation_project:AnnotationPodProjectDriver
 
 kuryr_kubernetes.controller.drivers.service_project =
     default = kuryr_kubernetes.controller.drivers.default_project:DefaultServiceProjectDriver
+    annotation = kuryr_kubernetes.controller.drivers.annotation_project:AnnotationServiceProjectDriver
 
 kuryr_kubernetes.controller.drivers.namespace_project =
     default = kuryr_kubernetes.controller.drivers.default_project:DefaultNamespaceProjectDriver
+    annotation = kuryr_kubernetes.controller.drivers.annotation_project:AnnotationNamespaceProjectDriver
 
 kuryr_kubernetes.controller.drivers.network_policy_project =
     default = kuryr_kubernetes.controller.drivers.default_project:DefaultNetworkPolicyProjectDriver
+    annotation = kuryr_kubernetes.controller.drivers.annotation_project:AnnotationNetworkPolicyProjectDriver
 
 kuryr_kubernetes.controller.drivers.pod_subnets =
     default = kuryr_kubernetes.controller.drivers.default_subnet:DefaultPodSubnetDriver