From 6f74b61518643d68fb4bb08531811b4d5edee6e9 Mon Sep 17 00:00:00 2001 From: Joao Soubihe Date: Tue, 5 Oct 2021 10:33:37 -0300 Subject: [PATCH] Preventing unlock and swact while k8s rootca update To avoid some possible errors tracking procedure states we decided to block unlock and swact host actions while certain phases of the procedure are being executed. For the reboot case we are considering that the option, in case of a failure due the host action is going to be an abort command execution and a new whole cycle of k8s rootca update procedure to set the cluster in an expected state. For this matter we've added couple methods that will validate the host actions mentioned above. Test Plan: Swact PASS: prevent swact during host update update-certs phase PASS: execute swact during kube rootca procedure but without any phase being executed PASS: execute swact after generate certs and proceed until the end of the procedure PASS: prevent swact during pods update Unlock PASS: Lock host after start and unlock host after update-cert. Then execute procedure until the end. PASS: Lock host after start and prevent unlock during k8s rootca update update-certs phase PASS: Lock host after start and prevent unlock during pods update phase trust-both-cas PASS: Lock and unlock host without k8s rootca update procedure ongoing PASS: Lock host while update phase in progress Reboot PASS: Cold shutdown host being updated. After turn on, abort procedure and complete a full cycle Closes-Bug: 1945329 Signed-off-by: Joao Soubihe Change-Id: Ifdd9bbf5802caba244d14e9c1de59cc71828ad04 --- .../sysinv/sysinv/api/controllers/v1/host.py | 89 +++++++++++++++++++ .../sysinv/sysinv/tests/api/test_host.py | 83 +++++++++++++++++ 2 files changed, 172 insertions(+) diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/host.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/host.py index 471124cae0..5cf6f7cdbb 100644 --- a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/host.py +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/host.py @@ -3513,6 +3513,46 @@ class HostController(rest.RestController): "Wait for kubelet upgrade to complete." % ihost['hostname']) raise wsme.exc.ClientSideError(msg) + def _semantic_check_unlock_kube_rootca_update(self, ihost, force_unlock=False): + """ + Perform semantic checks related to kubernetes rootca update + prior to unlocking host. + """ + + if force_unlock: + LOG.warning("Host %s force unlock while kubernetes " + "rootca update in progress." % ihost['hostname']) + return + + try: + kube_rootca_update = \ + pecan.request.dbapi.kube_rootca_update_get_one() + + if kube_rootca_update.state in [kubernetes.KUBE_ROOTCA_UPDATING_PODS_TRUSTBOTHCAS, + kubernetes.KUBE_ROOTCA_UPDATING_PODS_TRUSTNEWCA]: + msg = _("Can not unlock %s while kubernetes root ca " + "update phase in progress. Wait for update " + "phase to complete." % ihost['hostname']) + raise wsme.exc.ClientSideError(msg) + except exception.NotFound: + LOG.debug("No kubernetes rootca update was found") + return + + try: + kube_host_rootca_update = \ + pecan.request.dbapi.kube_rootca_host_update_get_by_host(ihost['uuid']) + if kube_host_rootca_update.state in [kubernetes.KUBE_ROOTCA_UPDATING_HOST_TRUSTBOTHCAS, + kubernetes.KUBE_ROOTCA_UPDATING_HOST_UPDATECERTS, + kubernetes.KUBE_ROOTCA_UPDATING_HOST_TRUSTNEWCA]: + msg = _("Can not unlock %s while kubernetes root ca " + "update phase in progress. Wait for update " + "phase to complete on host." % ihost['hostname']) + raise wsme.exc.ClientSideError(msg) + except exception.NotFound: + LOG.debug("No kubernetes rootca update on host %s " + "was found" % ihost['hostname']) + return + def _semantic_check_unlock_upgrade(self, ihost, force_unlock=False): """ Perform semantic checks related to upgrades prior to unlocking host. @@ -5376,6 +5416,10 @@ class HostController(rest.RestController): # the unlock. self.check_unlock_application(hostupdate, force_unlock) + # Ensure there is no k8s rootca update phase in progress + self._semantic_check_unlock_kube_rootca_update(hostupdate.ihost_orig, + force_unlock) + personality = hostupdate.ihost_patch.get('personality') if personality == constants.CONTROLLER: self.check_unlock_controller(hostupdate, force_unlock) @@ -6037,6 +6081,47 @@ class HostController(rest.RestController): # Check for new hardware since upgrade-start self._semantic_check_upgrade_refresh(upgrade, to_host, force_swact) + def _semantic_check_swact_kube_rootca_update(self, ihost, force_swact=False): + """ + Perform semantic checks related to kubernetes rootca update + prior to swacting host. + """ + + if force_swact: + LOG.warning("Host %s force swact while kubernetes " + "rootca update in progress on host." + % ihost['hostname']) + return + + try: + kube_rootca_update = \ + pecan.request.dbapi.kube_rootca_update_get_one() + + if kube_rootca_update.state in [kubernetes.KUBE_ROOTCA_UPDATING_PODS_TRUSTBOTHCAS, + kubernetes.KUBE_ROOTCA_UPDATING_PODS_TRUSTNEWCA]: + msg = _("Can not swact %s while kubernetes root ca " + "update phase in progress. Wait for update " + "phase to complete." % ihost['hostname']) + raise wsme.exc.ClientSideError(msg) + except exception.NotFound: + LOG.debug("No kubernetes rootca update was found") + return + + try: + kube_host_rootca_update = \ + pecan.request.dbapi.kube_rootca_host_update_get_by_host(ihost['uuid']) + if kube_host_rootca_update.state in [kubernetes.KUBE_ROOTCA_UPDATING_HOST_TRUSTBOTHCAS, + kubernetes.KUBE_ROOTCA_UPDATING_HOST_UPDATECERTS, + kubernetes.KUBE_ROOTCA_UPDATING_HOST_TRUSTNEWCA]: + msg = _("Can not swact %s while kubernetes root ca " + "update phase is in progress. Wait for update " + "phase to complete on host." % ihost['hostname']) + raise wsme.exc.ClientSideError(msg) + except exception.NotFound: + LOG.debug("No kubernetes rootca update on host %s " + "was found" % ihost['hostname']) + return + def _check_swact_device_image_update(self, from_host, to_host, force=False): if force: LOG.info("device image update swact check bypassed with force option") @@ -6114,6 +6199,10 @@ class HostController(rest.RestController): ihost_ctr.subfunction_oper, ihost_ctr.subfunctions)) + # deny swact if a kube rootca update phase is in progress + self._semantic_check_swact_kube_rootca_update(hostupdate.ihost_orig, + force_swact) + # deny swact if storage backend not ready self._semantic_check_storage_backend(ihost_ctr) diff --git a/sysinv/sysinv/sysinv/sysinv/tests/api/test_host.py b/sysinv/sysinv/sysinv/sysinv/tests/api/test_host.py index 6af87bea2d..6eafe87d7a 100644 --- a/sysinv/sysinv/sysinv/sysinv/tests/api/test_host.py +++ b/sysinv/sysinv/sysinv/sysinv/tests/api/test_host.py @@ -1995,6 +1995,27 @@ class TestPatch(TestHost): self.assertEqual(http_client.BAD_REQUEST, response.status_int) self.assertTrue(response.json['error_message']) + def test_unlock_action_controller_during_k8s_rootca_pods_update(self): + # Create controller-0 without inv_state initial inventory complete + c0_host = self._create_controller_0( + invprovision=constants.PROVISIONED, + administrative=constants.ADMIN_LOCKED, + operational=constants.OPERATIONAL_ENABLED, + availability=constants.AVAILABILITY_ONLINE, + inv_state=None, clock_synchronization=constants.NTP) + + # Create kube rootca update updating pods on phase trust-both-cas + dbutils.create_test_kube_rootca_update(state=kubernetes.KUBE_ROOTCA_UPDATING_PODS_TRUSTBOTHCAS) + + # Unlock host + response = self._patch_host_action(c0_host['hostname'], + constants.UNLOCK_ACTION, + 'sysinv-test', + expect_errors=True) + self.assertEqual(response.content_type, 'application/json') + self.assertEqual(http_client.BAD_REQUEST, response.status_int) + self.assertTrue(response.json['error_message']) + def _test_lock_action_controller(self): # Create controller-0 self._create_controller_0( @@ -2790,6 +2811,68 @@ class TestPatchStdDuplexControllerAction(TestHost): "images." % (c0_host['hostname'], c1_host['hostname']), response.json['error_message']) + def test_swact_action_controller_while_kube_rootca_pods_update(self): + # Create controller-0 + c0_host = self._create_controller_0( + invprovision=constants.PROVISIONED, + administrative=constants.ADMIN_UNLOCKED, + operational=constants.OPERATIONAL_ENABLED, + availability=constants.AVAILABILITY_ONLINE) + + self._create_controller_1( + invprovision=constants.PROVISIONED, + administrative=constants.ADMIN_UNLOCKED, + operational=constants.OPERATIONAL_ENABLED, + availability=constants.AVAILABILITY_ONLINE) + + # Create kube rootca update updating pods on phase trust-both-cas + dbutils.create_test_kube_rootca_update(state=kubernetes.KUBE_ROOTCA_UPDATING_PODS_TRUSTBOTHCAS) + + # Swact controller host + response = self._patch_host_action(c0_host['hostname'], + constants.SWACT_ACTION, + 'sysinv-test', + expect_errors=True) + self.assertEqual(response.content_type, 'application/json') + self.assertEqual(http_client.BAD_REQUEST, response.status_int) + self.assertTrue(response.json['error_message']) + self.assertIn("Can not swact %s while kubernetes root ca " + "update phase in progress. Wait for update " + "phase to complete." % c0_host['hostname'], + response.json['error_message']) + + def test_swact_action_controller_while_kube_rootca_host_update(self): + # Create controller-0 + c0_host = self._create_controller_0( + invprovision=constants.PROVISIONED, + administrative=constants.ADMIN_UNLOCKED, + operational=constants.OPERATIONAL_ENABLED, + availability=constants.AVAILABILITY_ONLINE) + + self._create_controller_1( + invprovision=constants.PROVISIONED, + administrative=constants.ADMIN_UNLOCKED, + operational=constants.OPERATIONAL_ENABLED, + availability=constants.AVAILABILITY_ONLINE) + + # Create kubernetes rootca update for the host and set it with phase in progress + dbutils.create_test_kube_rootca_update(state=kubernetes.KUBE_ROOTCA_UPDATING_HOST_UPDATECERTS) + dbutils.create_test_kube_rootca_host_update(host_id=c0_host['id'], + state=kubernetes.KUBE_ROOTCA_UPDATING_HOST_UPDATECERTS) + + # Swact controller host + response = self._patch_host_action(c0_host['hostname'], + constants.SWACT_ACTION, + 'sysinv-test', + expect_errors=True) + self.assertEqual(response.content_type, 'application/json') + self.assertEqual(http_client.BAD_REQUEST, response.status_int) + self.assertTrue(response.json['error_message']) + self.assertIn("Can not swact %s while kubernetes root ca " + "update phase is in progress. Wait for update " + "phase to complete on host." % c0_host['hostname'], + response.json['error_message']) + class TestPatchStdDuplexControllerVIM(TestHost):