Merge "Update system:node clusterrolebinding for new host"

sysinv/sysinv/sysinv/sysinv/common/kubernetes.py

@@ -374,6 +374,7 @@ class KubeOperator(object):
         self._kube_client_policy = None
         self._kube_client_custom_objects = None
         self._kube_client_admission_registration = None
+        self._kube_client_rbac_authorization = None
 
     def _load_kube_config(self):
         if not is_k8s_configured():
@@ -422,6 +423,12 @@ class KubeOperator(object):
             self._kube_client_admission_registration = client.AdmissionregistrationV1Api()
         return self._kube_client_admission_registration
 
+    def _get_kubernetesclient_rbac_authorization(self):
+        if not self._kube_client_rbac_authorization:
+            self._load_kube_config()
+            self._kube_client_rbac_authorization = client.RbacAuthorizationV1Api()
+        return self._kube_client_rbac_authorization
+
     def _retry_on_urllibs3_MaxRetryError(ex):  # pylint: disable=no-self-argument
         if isinstance(ex, MaxRetryError):
             LOG.warn('Retrying against MaxRetryError: {}'.format(ex))
@@ -1554,3 +1561,30 @@ class KubeOperator(object):
         except Exception as e:
             LOG.exception("Failed to fetch PodSecurityPolicies: %s" % e)
             raise
+
+    def kube_read_clusterrolebinding(self, name):
+        """read a clusterrolebinding with data
+
+        """
+        try:
+            rbac_authorization = self._get_kubernetesclient_rbac_authorization()
+            v1_cluster_role_binding_object = rbac_authorization.read_cluster_role_binding(name)
+            LOG.info("Clusterrolebinding %s retrieved successfully." % name)
+            return v1_cluster_role_binding_object
+        except Exception as ex:
+            LOG.exception("Failed to read clusterolebinding %s : %s" % (name, ex))
+            raise
+
+    def kube_patch_clusterrolebinding(self, name, body):
+        """patch a clusterrolebinding with data
+
+        """
+        try:
+            rbac_authorization = self._get_kubernetesclient_rbac_authorization()
+            v1_cluster_role_binding_object = \
+                    rbac_authorization.patch_cluster_role_binding(name, body)
+            LOG.info("Clusterrolebinding %s updated successfully. "
+                     "Updated object: %s" % (name, v1_cluster_role_binding_object))
+        except Exception as ex:
+            LOG.exception("Failed to patch clusterolebinding %s : %s" % (name, ex))
+            raise

sysinv/sysinv/sysinv/sysinv/conductor/manager.py

@@ -55,6 +55,7 @@ from datetime import datetime
 from datetime import timedelta
 from distutils.version import LooseVersion
 from copy import deepcopy
+from urllib3.exceptions import MaxRetryError
 
 import tsconfig.tsconfig as tsc
 from collections import namedtuple
@@ -1113,6 +1114,70 @@ class ConductorManager(service.PeriodicService):
                 return host
         return None
 
+    def _retry_on_patch_system_node_clusterrolebinding(ex):  # pylint: disable=no-self-argument
+        if isinstance(ex, MaxRetryError):
+            LOG.warning("system:node clusterrolebinding patch unsuccessful. Retrying...")
+            return True
+        else:
+            return False
+
+    @retry(stop_max_attempt_number=4,
+           wait_fixed=15 * 1000,
+           retry_on_exception=_retry_on_patch_system_node_clusterrolebinding)
+    def _system_node_clusterrolebinding_add_host(self, hostname):
+        """Adds new host to the system:node clusterrolebinding
+
+        This method adds an entry of the new host as a subject to the
+        system:node clusterrolebinding.
+
+        :param hostname: name of the host to be added
+        """
+        try:
+            subject = {
+                'api_group': 'rbac.authorization.k8s.io',
+                'kind': 'User',
+                'name': 'system:node:%s' % hostname,
+                'namespace': None
+            }
+            v1_cluster_role_binding_object = self._kube.kube_read_clusterrolebinding("system:node")
+            # As this code is also run during upgrade-activate operation,
+            # we must ensure that it does not create multiple entries for the same host
+            # if the upgrade-activate operation is re-run.
+            if not any(subject.name == ["system:node:%s" % hostname]
+                       for subject in v1_cluster_role_binding_object.subjects):
+                v1_cluster_role_binding_object.subjects.append(subject)
+            self._kube.kube_patch_clusterrolebinding("system:node", v1_cluster_role_binding_object)
+            LOG.info("Host system:node:%s was added as a subject to the 'system:node' "
+                     "clusterrolebinding" % hostname)
+        except Exception as ex:
+            LOG.error("Failed to add host system:node:%s as a subject to the 'system:node' "
+                      "clusterrolebinding with error: %s" % (hostname, ex))
+            raise
+
+    @retry(stop_max_attempt_number=4,
+           wait_fixed=15 * 1000,
+           retry_on_exception=_retry_on_patch_system_node_clusterrolebinding)
+    def _system_node_clusterrolebinding_remove_host(self, hostname):
+        """Remove host from the system:node clusterrolebinding
+
+        This method removes host entry from the subjects list in the
+        system:node clusterrolebinding.
+
+        :param hostname: name of the host to be removed
+        """
+        try:
+            v1_cluster_role_binding_object = self._kube.kube_read_clusterrolebinding("system:node")
+            subjects = v1_cluster_role_binding_object.subjects
+            subjects[:] = [subject for subject in subjects
+                            if subject.name != 'system:node:%s' % hostname]
+            self._kube.kube_patch_clusterrolebinding("system:node", v1_cluster_role_binding_object)
+            LOG.info("Host system:node:%s was removed from subjects in the 'system:node' "
+                    "clusterrolebinding" % hostname)
+        except Exception as ex:
+            LOG.error("Failed to remove host system:node:%s from subjects in the 'system:node' "
+                    "clusterrolebinding with error: %s" % (hostname, ex))
+            raise
+
     def create_ihost(self, context, values, reason=None):
         """Create an ihost with the supplied data.
 
@@ -1165,6 +1230,20 @@ class ConductorManager(service.PeriodicService):
 
         ihost = self.dbapi.ihost_create(values, software_load=software_load)
 
+        try:
+            hostname = values.get("hostname")
+            # As storage hosts don't run kubelet, we do not add them to the
+            # clusterrolebinding. Also, as kubernetes is not up while
+            # adding controller-0 during ansible bootstrap, we skip calling
+            # this method for controller-0 which is handled in the ansible
+            # code.
+            if hostname and \
+                    not os.path.isfile(constants.ANSIBLE_BOOTSTRAP_FLAG) and \
+                        values.get('personality') != constants.STORAGE:
+                self._system_node_clusterrolebinding_add_host(hostname)
+        except Exception as ex:
+            LOG.error("Error adding host to the system:node clusterrolebinding: %s" % ex)
+
         # A host is being created, generate discovery log.
         self._log_host_create(ihost, reason)
 
@@ -2873,6 +2952,16 @@ class ConductorManager(service.PeriodicService):
                 # allow a host with no personality to be unconfigured
                 pass
 
+        try:
+            # Storage hosts don't run kubelet and are not added to the
+            # clusterrolebinding.
+            if ihost_obj.personality != constants.STORAGE:
+                self._system_node_clusterrolebinding_remove_host(ihost_obj.hostname)
+        except Exception as ex:
+            # As this is just a cleanup operation we do nothing than just
+            # logging the error.
+            LOG.error("Error removing host from the clusterrolebinding: %s" % ex)
+
     def _update_dependent_interfaces(self, interface, ihost,
                                      phy_intf, oldmac, newmac, depth=1):
         """ Updates the MAC address for dependent logical interfaces.

sysinv/sysinv/sysinv/sysinv/tests/common/test_kubernetes.py

@@ -767,6 +767,24 @@ class TestKubeOperator(base.TestCase):
             'items': []
         }
 
+        self.read_clusterrolebinding_result = kubernetes.client.V1ClusterRoleBinding(
+            api_version="rbac.authorization.k8s.io/v1",
+            kind="ClusterRoleBinding",
+            metadata=kubernetes.client.V1ObjectMeta(
+                name="test_system:test_node",
+            ),
+            role_ref=kubernetes.client.V1RoleRef(
+                api_group='rbac.authorization.k8s.io',
+                kind='ClusterRole',
+                name='test_system:test_node'
+            ),
+            subjects=[kubernetes.client.V1Subject(
+                kind='User',
+                name='test_system:test_node:test_hostname',
+                api_group='rbac.authorization.k8s.io',
+            )],
+        )
+
     def setUp(self):
         super(TestKubeOperator, self).setUp()
 
@@ -841,6 +859,13 @@ class TestKubeOperator(base.TestCase):
             mock_list_pod_security_policy)
         self.mocked_list_pod_security_policy.start()
 
+        def mock_read_clusterrolebinding(obj, name):
+            return self.read_clusterrolebinding_result
+        self.mocked_read_clusterrolebinding = mock.patch(
+            'kubernetes.client.RbacAuthorizationV1Api.read_cluster_role_binding',
+            mock_read_clusterrolebinding)
+        self.mocked_read_clusterrolebinding.start()
+
         self.kube_operator = kube.KubeOperator()
 
     def tearDown(self):
@@ -853,6 +878,7 @@ class TestKubeOperator(base.TestCase):
         self.mocked_read_namespaced_service_account.stop()
         self.mocked_read_namespaced_secret.stop()
         self.mocked_list_pod_security_policy.stop()
+        self.mocked_read_clusterrolebinding.stop()
 
     def test_kube_get_image_by_pod_name(self):
 
@@ -1335,6 +1361,58 @@ class TestKubeOperator(base.TestCase):
         mock_kube_patch_config_map.assert_called_with(
                 'kubeadm-config', 'kube-system', patch_config_map_arg)
 
+    def test_read_clusterrolebinding_success(self):
+        mock_read_clusterrolebinding = mock.MagicMock()
+        mocked_read_clusterrolebinding = mock.patch(
+            'kubernetes.client.RbacAuthorizationV1Api.read_cluster_role_binding',
+            mock_read_clusterrolebinding
+        )
+        mocked_read_clusterrolebinding.start().return_value = self.read_clusterrolebinding_result
+        self.addCleanup(mocked_read_clusterrolebinding.stop)
+
+        fake_clusterrole_binding_name = "test_system:test_node"
+        result = self.kube_operator.kube_read_clusterrolebinding(fake_clusterrole_binding_name)
+        mock_read_clusterrolebinding.assert_called_with(fake_clusterrole_binding_name)
+        self.assertEqual(result, self.read_clusterrolebinding_result)
+
+    def test_read_clusterrolebinding_exception(self):
+        mock_read_clusterrolebinding = mock.MagicMock()
+        mocked_read_clusterrolebinding = mock.patch(
+            'kubernetes.client.RbacAuthorizationV1Api.read_cluster_role_binding',
+            mock_read_clusterrolebinding
+        )
+        mocked_read_clusterrolebinding.start().side_effect = Exception("Fake Error")
+        self.addCleanup(mocked_read_clusterrolebinding.stop)
+        self.assertRaises(  # noqa: H202
+            Exception,
+            self.kube_operator.kube_read_clusterrolebinding,
+        )
+
+    def test_patch_clusterrolebinding_success(self):
+        mock_patch_clusterrolebinding = mock.MagicMock()
+        mocked_patch_clusterrolebinding = mock.patch(
+            'kubernetes.client.RbacAuthorizationV1Api.patch_cluster_role_binding',
+            mock_patch_clusterrolebinding
+        )
+        mocked_patch_clusterrolebinding.start().return_value = self.read_clusterrolebinding_result
+        self.addCleanup(mocked_patch_clusterrolebinding.stop)
+        fake_clusterrole_binding_name = "test_system:test_node"
+        self.kube_operator.kube_patch_clusterrolebinding(fake_clusterrole_binding_name, {})
+        mock_patch_clusterrolebinding.assert_called_with(fake_clusterrole_binding_name, {})
+
+    def test_patch_clusterrolebinding_exception(self):
+        mock_patch_clusterrolebinding = mock.MagicMock()
+        mocked_patch_clusterrolebinding = mock.patch(
+            'kubernetes.client.RbacAuthorizationV1Api.patch_cluster_role_binding',
+            mock_patch_clusterrolebinding
+        )
+        mocked_patch_clusterrolebinding.start().side_effect = Exception("Fake Error")
+        self.addCleanup(mocked_patch_clusterrolebinding.stop)
+        self.assertRaises(     # noqa: H202
+            Exception,
+            self.kube_operator.kube_patch_clusterrolebinding,
+        )
+
 
 class TestKubernetesUtilities(base.TestCase):
     def test_is_kube_version_supported(self):

sysinv/sysinv/sysinv/sysinv/tests/conductor/test_manager.py

@@ -488,6 +488,40 @@ class ManagerTestCase(base.DbTestCase):
         self.mocked_get_kube_versions.start()
         self.addCleanup(self.mocked_get_kube_versions.stop)
 
+        self.kube_read_clusterrolebinding_result = kubernetes.client.V1ClusterRoleBinding(
+            api_version="rbac.authorization.k8s.io/v1",
+            kind="ClusterRoleBinding",
+            metadata=kubernetes.client.V1ObjectMeta(
+                name="test_system:test_node",
+            ),
+            role_ref=kubernetes.client.V1RoleRef(
+                api_group='rbac.authorization.k8s.io',
+                kind='ClusterRole',
+                name='test_system:test_node'
+            ),
+            subjects=[kubernetes.client.V1Subject(
+                kind='User',
+                name='test_system:test_node:test_hostname',
+                api_group='rbac.authorization.k8s.io',
+            )],
+        )
+
+        mock_kube_read_clusterrolebinding = mock.MagicMock()
+        self.mocked_kube_read_clusterrolebinding = mock.patch(
+            'sysinv.common.kubernetes.KubeOperator.kube_read_clusterrolebinding',
+            mock_kube_read_clusterrolebinding)
+        self.mocked_kube_read_clusterrolebinding.start().return_value = self.kube_read_clusterrolebinding_result
+        self.addCleanup(self.mocked_kube_read_clusterrolebinding.stop)
+        self.mock_kube_read_clusterrolebinding = mock_kube_read_clusterrolebinding
+
+        mock_kube_patch_clusterrolebinding = mock.MagicMock()
+        self.mocked_kube_patch_clusterrolebinding = mock.patch(
+            'sysinv.common.kubernetes.KubeOperator.kube_patch_clusterrolebinding',
+            mock_kube_patch_clusterrolebinding)
+        self.mocked_kube_patch_clusterrolebinding.start().return_value = self.kube_read_clusterrolebinding_result
+        self.addCleanup(self.mocked_kube_patch_clusterrolebinding.stop)
+        self.mock_kube_patch_clusterrolebinding = mock_kube_patch_clusterrolebinding
+
         self.service._puppet = mock.Mock()
         self.service._allocate_addresses_for_host = mock.Mock()
         self.service._update_pxe_config = mock.Mock()
@@ -611,6 +645,73 @@ class ManagerTestCase(base.DbTestCase):
         for k, v in ihost_dict.items():
             self.assertEqual(res[k], v)
 
+    def test_system_node_clusterrolebinding_add_host_success(self):
+        self.service.start()
+        self.service._system_node_clusterrolebinding_add_host(
+                "test_system:test_node:test_hostname-1")
+        self.mock_kube_read_clusterrolebinding.assert_called()
+        self.mock_kube_patch_clusterrolebinding.assert_called()
+
+    def test_system_node_clusterrolebinding_add_host_already_exists(self):
+        mock_kube_read_clusterrolebinding = mock.MagicMock()
+        mocked_kube_read_clusterrolebinding = mock.patch(
+            'sysinv.common.kubernetes.KubeOperator.kube_read_clusterrolebinding',
+            mock_kube_read_clusterrolebinding
+        )
+        mocked_kube_read_clusterrolebinding.start().return_value = \
+            self.kube_read_clusterrolebinding_result
+        self.addCleanup(mocked_kube_read_clusterrolebinding.stop)
+
+        mock_kube_patch_clusterrolebinding = mock.MagicMock()
+        mocked_kube_patch_clusterrolebinding = mock.patch(
+            'sysinv.common.kubernetes.KubeOperator.kube_patch_clusterrolebinding',
+            mock_kube_patch_clusterrolebinding
+        )
+        mocked_kube_patch_clusterrolebinding.start()
+        self.addCleanup(mocked_kube_patch_clusterrolebinding.stop)
+
+        self.service.start()
+        self.service._system_node_clusterrolebinding_add_host(
+            "test_system:test_node:test_hostname")
+        mock_kube_read_clusterrolebinding.assert_called()
+        mock_kube_patch_clusterrolebinding.assert_called_with(
+            "system:node", self.kube_read_clusterrolebinding_result)
+
+    def test_system_node_clusterrolebinding_remove_host_success(self):
+        self.service.start()
+        self.service._system_node_clusterrolebinding_remove_host(
+                "test_system:test_node:test:hostname")
+        self.mock_kube_read_clusterrolebinding.assert_called()
+        self.mock_kube_patch_clusterrolebinding.assert_called()
+
+    def test_system_node_clusterrolebinding_add_host_exception(self):
+        mock_kube_read_clusterrolebinding = mock.MagicMock()
+        mocked_kube_read_clusterrolebinding = mock.patch(
+            'sysinv.common.kubernetes.KubeOperator.kube_read_clusterrolebinding',
+            mock_kube_read_clusterrolebinding
+        )
+        mocked_kube_read_clusterrolebinding.start().side_effect = Exception("Fake Error")
+        self.addCleanup(mocked_kube_read_clusterrolebinding.stop)
+        self.service.start()
+        self.assertRaises(     # noqa: H202
+            Exception,
+            self.service._system_node_clusterrolebinding_add_host,
+        )
+
+    def test_system_node_clusterrolebinding_remove_host_exception(self):
+        mock_kube_patch_clusterrolebinding = mock.MagicMock()
+        mocked_kube_patch_clusterrolebinding = mock.patch(
+            'sysinv.common.kubernetes.KubeOperator.kube_patch_clusterrolebinding',
+            mock_kube_patch_clusterrolebinding
+        )
+        mocked_kube_patch_clusterrolebinding.start().side_effect = Exception("Fake Error")
+        self.addCleanup(mocked_kube_patch_clusterrolebinding.stop)
+        self.service.start()
+        self.assertRaises(     # noqa: H202
+            Exception,
+            self.service._system_node_clusterrolebinding_remove_host,
+        )
+
     def test_update_ihost(self):
         ihost = self._create_test_ihost()