Merge "Re-introduce kube upgrade health check for duplex systems"

nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py

@@ -468,9 +468,11 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
 
                 host_label_list = future.result.data["labels"]
 
-                openstack_compute, openstack_control, remote_storage = (
-                    self._get_host_labels(host_label_list)
-                )
+                (
+                    openstack_compute,
+                    openstack_control,
+                    remote_storage,
+                ) = self._get_host_labels(host_label_list)
 
                 admin_state, oper_state, avail_status, nfvi_data = host_state(
                     host_uuid,
@@ -641,9 +643,11 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
 
             host_label_list = future.result.data["labels"]
 
-            openstack_compute, openstack_control, remote_storage = (
-                self._get_host_labels(host_label_list)
-            )
+            (
+                openstack_compute,
+                openstack_control,
+                remote_storage,
+            ) = self._get_host_labels(host_label_list)
 
             host_obj = nfvi.objects.v1.Host(
                 host_uuid,
@@ -3538,9 +3542,11 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
 
             host_label_list = future.result.data["labels"]
 
-            openstack_compute, openstack_control, remote_storage = (
-                self._get_host_labels(host_label_list)
-            )
+            (
+                openstack_compute,
+                openstack_control,
+                remote_storage,
+            ) = self._get_host_labels(host_label_list)
 
             host_obj = nfvi.objects.v1.Host(
                 host_uuid,
@@ -3673,9 +3679,11 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
 
             host_label_list = future.result.data["labels"]
 
-            openstack_compute, openstack_control, remote_storage = (
-                self._get_host_labels(host_label_list)
-            )
+            (
+                openstack_compute,
+                openstack_control,
+                remote_storage,
+            ) = self._get_host_labels(host_label_list)
 
             host_obj = nfvi.objects.v1.Host(
                 host_uuid,
@@ -3811,9 +3819,11 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
 
             host_label_list = future.result.data["labels"]
 
-            openstack_compute, openstack_control, remote_storage = (
-                self._get_host_labels(host_label_list)
-            )
+            (
+                openstack_compute,
+                openstack_control,
+                remote_storage,
+            ) = self._get_host_labels(host_label_list)
 
             host_obj = nfvi.objects.v1.Host(
                 host_uuid,
@@ -3950,9 +3960,11 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
 
             host_label_list = future.result.data["labels"]
 
-            openstack_compute, openstack_control, remote_storage = (
-                self._get_host_labels(host_label_list)
-            )
+            (
+                openstack_compute,
+                openstack_control,
+                remote_storage,
+            ) = self._get_host_labels(host_label_list)
 
             host_obj = nfvi.objects.v1.Host(
                 host_uuid,
@@ -4144,9 +4156,11 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
 
             host_label_list = future.result.data["labels"]
 
-            openstack_compute, openstack_control, remote_storage = (
-                self._get_host_labels(host_label_list)
-            )
+            (
+                openstack_compute,
+                openstack_control,
+                remote_storage,
+            ) = self._get_host_labels(host_label_list)
 
             host_obj = nfvi.objects.v1.Host(
                 host_uuid,
@@ -4893,6 +4907,59 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
             callback.send(response)
             callback.close()
 
+    def get_kube_upgrade_health(self, future, alarm_ignore_list, callback):
+        """Get kube upgrade health status from the plugin."""
+
+        response = {}
+        response["completed"] = False
+        response["reason"] = ""
+        action_type = "get-kube-upgrade-health"
+
+        try:
+            future.set_timeouts(config.CONF.get("nfvi-timeouts", None))
+
+            if self._platform_token is None or self._platform_token.is_expired():
+                future.work(openstack.get_token, self._platform_directory)
+                future.result = yield
+
+                if not future.result.is_complete() or future.result.data is None:
+                    DLOG.error("OpenStack get-token did not complete.")
+                    return
+
+                self._platform_token = future.result.data
+
+            future.work(
+                sysinv.get_kube_upgrade_health, self._platform_token, alarm_ignore_list
+            )
+            future.result = yield
+
+            if not future.result.is_complete():
+                DLOG.error("SysInv get-kube-upgrade-health did not complete.")
+                return
+
+            response["result-data"] = future.result.data
+            response["completed"] = True
+
+        except exceptions.OpenStackRestAPIException as e:
+            if httplib.UNAUTHORIZED == e.http_status_code:
+                response["error-code"] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED
+                if self._platform_token is not None:
+                    self._platform_token.set_expired()
+
+            else:
+                DLOG.exception(
+                    "Caught API exception while trying %s. error=%s" % (action_type, e)
+                )
+
+        except Exception as e:
+            DLOG.exception(
+                "Caught exception while trying %s. error=%s" % (action_type, e)
+            )
+
+        finally:
+            callback.send(response)
+            callback.close()
+
     def get_deployment_host(self, future, host_name, callback):
         """Get a host resource from the deployment namespace space."""
 

nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/sysinv.py

@@ -231,6 +231,23 @@ def get_kube_upgrade(token):
     return response
 
 
+def get_kube_upgrade_health(token, alarm_ignore_list=None):
+    """Asks System Inventory for kube upgrade health status."""
+
+    url = token.get_service_url(PLATFORM_SERVICE.SYSINV)
+    if url is None:
+        raise ValueError("OpenStack SysInv URL is invalid")
+
+    api_cmd = url + "/health/kube-upgrade"
+    if alarm_ignore_list:
+        api_cmd += "?alarm_ignore_list=%s" % ",".join(alarm_ignore_list)
+
+    response = rest_api_request(
+        token, "GET", api_cmd, timeout_in_secs=REST_API_REQUEST_TIMEOUT
+    )
+    return response
+
+
 def get_kube_version(token, kube_version):
     """Asks System Inventory for information a kube version."""
 

nfv/nfv-tests/nfv_unit_tests/tests/test_sw_deploy_strategy.py

@@ -4972,12 +4972,12 @@ class TestSwUpgradeCombinedKubeStrategy(BaseSwUpgradeStrategy):
         self.assertLess(sw_start_idx, kube_start_idx)
         self.assertLess(sw_complete_idx, kube_start_idx)
 
-        # kube-wait-control-plane-pods-ready must appear
+        # kube-wait-upgrade-healthy must appear
         # between sw-deploy and kube-upgrade
-        self.assertIn("kube-wait-control-plane-pods-ready", stage_names)
-        pods_ready_idx = stage_names.index("kube-wait-control-plane-pods-ready")
-        self.assertLess(sw_complete_idx, pods_ready_idx)
-        self.assertLess(pods_ready_idx, kube_start_idx)
+        self.assertIn("kube-wait-upgrade-healthy", stage_names)
+        upgrade_healthy_idx = stage_names.index("kube-wait-upgrade-healthy")
+        self.assertLess(sw_complete_idx, upgrade_healthy_idx)
+        self.assertLess(upgrade_healthy_idx, kube_start_idx)
 
         # No sw-system-deploy-init (duplex does not use it)
         self.assertNotIn("sw-system-deploy-init", stage_names)
@@ -6446,3 +6446,331 @@ class TestWaitKubeControlPlanePodsReadyStep(
         self.mock_stage.step_complete.assert_called_once_with(
             common_strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
         )
+
+
+@mock.patch(
+    "nfv_vim.event_log._instance._event_issue", sw_update_testcase.fake_event_issue
+)
+@mock.patch("nfv_vim.objects._sw_update.SwUpdate.save", sw_update_testcase.fake_save)
+@mock.patch(
+    "nfv_vim.objects._sw_update.timers.timers_create_timer",
+    sw_update_testcase.fake_timer,
+)
+@mock.patch(
+    "nfv_vim.nfvi.nfvi_compute_plugin_disabled",
+    sw_update_testcase.fake_nfvi_compute_plugin_disabled,
+)
+class TestWaitKubernetesUpgradeHealthy(sw_update_testcase.SwUpdateStrategyTestCase):
+    """Unit tests for the WaitKubernetesUpgradeHealthy step."""
+
+    def setUp(self):
+        super().setUp()
+        from nfv_vim.strategy.steps.kube_upgrade_steps import (
+            WaitKubernetesUpgradeHealthy,
+        )
+
+        self.step = WaitKubernetesUpgradeHealthy(timeout_in_secs=180)
+        self.mock_stage = mock.MagicMock()
+        self.step.stage = self.mock_stage
+
+    def _send_callback(self, response):
+        """Drive the coroutine callback with the given response."""
+
+        callback = self.step._query_health_callback()
+        try:
+            callback.send(response)
+        except StopIteration:
+            pass
+
+    def test_healthy_completes_step_successfully(self):
+        """Step completes with SUCCESS when health check has no failures."""
+
+        self._send_callback(
+            {
+                "completed": True,
+                "result-data": "OK: all checks passed",
+            }
+        )
+
+        self.mock_stage.step_complete.assert_called_once_with(
+            common_strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
+        )
+
+    def test_fail_in_health_does_not_complete_step(self):
+        """Step keeps waiting when health check contains [Fail] entries."""
+
+        self._send_callback(
+            {
+                "completed": True,
+                "result-data": "[Fail] Kubernetes nodes not ready",
+            }
+        )
+
+        self.mock_stage.step_complete.assert_not_called()
+
+    def test_query_failure_does_not_fail_step(self):
+        """Step retries when the query itself fails (does not fail step)."""
+
+        self._send_callback({"completed": False, "reason": "connection error"})
+
+        self.mock_stage.step_complete.assert_not_called()
+
+    def test_missing_result_data_fails_step(self):
+        """Step fails when response is completed but result-data is missing."""
+
+        self._send_callback({"completed": True})
+
+        self.mock_stage.step_complete.assert_called_once_with(
+            common_strategy.STRATEGY_STEP_RESULT.FAILED,
+            "Kube upgrade health check missing result-data",
+        )
+
+    def test_apply_creates_timer_and_returns_wait(self):
+        """Apply creates a polling timer and returns WAIT."""
+
+        from nfv_common import timers
+
+        with mock.patch.object(
+            timers, "timers_create_timer", return_value=999
+        ) as mock_create:
+            result, reason = self.step.apply()
+
+        self.assertEqual(result, common_strategy.STRATEGY_STEP_RESULT.WAIT)
+        self.assertEqual(reason, "")
+        mock_create.assert_called_once()
+        self.assertEqual(self.step._poll_timer_id, 999)
+
+    def test_handle_event_returns_false(self):
+        """handle_event is a no-op (polling is timer-driven)."""
+
+        from nfv_vim.strategy._strategy_defs import STRATEGY_EVENT
+
+        result = self.step.handle_event(STRATEGY_EVENT.HOST_AUDIT)
+        self.assertFalse(result)
+
+    def test_poll_action_triggers_query_with_alarm_ignore_list(self):
+        """_poll_action calls the NFVI query with alarm_ignore_list."""
+
+        from nfv_vim.strategy.steps.kube_upgrade_steps import (
+            KUBE_UPGRADE_START_ALARM_IGNORE,
+        )
+
+        with mock.patch("nfv_vim.nfvi.nfvi_get_kube_upgrade_health") as mock_query:
+            self.step._poll_action()
+            mock_query.assert_called_once()
+            self.assertTrue(self.step._poll_in_progress)
+            # Verify alarm_ignore_list is passed as first argument
+            call_args = mock_query.call_args[0]
+            self.assertEqual(call_args[0], KUBE_UPGRADE_START_ALARM_IGNORE)
+
+    def test_poll_skipped_when_in_progress(self):
+        """Timer callback does not trigger a new query while one is in progress."""
+
+        from nfv_common import timers
+
+        with mock.patch.object(timers, "timers_create_timer", return_value=999):
+            self.step.apply()
+
+        # Simulate first poll
+        with mock.patch("nfv_vim.nfvi.nfvi_get_kube_upgrade_health") as mock_query:
+            self.step._poll_action()
+            mock_query.assert_called_once()
+
+        # poll_in_progress is True, so _poll_timer_callback should not call again
+        self.assertTrue(self.step._poll_in_progress)
+
+    def test_from_dict_roundtrip(self):
+        """Step can be serialized and deserialized without data loss."""
+
+        from nfv_vim.strategy.steps.kube_upgrade_steps import (
+            KUBE_UPGRADE_START_ALARM_IGNORE,
+            WaitKubernetesUpgradeHealthy,
+        )
+
+        data = self.step.as_dict()
+        self.assertEqual(data["alarm_ignore_list"], KUBE_UPGRADE_START_ALARM_IGNORE)
+
+        new_step = object.__new__(WaitKubernetesUpgradeHealthy)
+        new_step.from_dict(data)
+
+        self.assertIsNone(new_step._poll_timer_id)
+        self.assertFalse(new_step._poll_in_progress)
+        self.assertEqual(new_step._alarm_ignore_list, KUBE_UPGRADE_START_ALARM_IGNORE)
+
+    def test_from_dict_roundtrip_custom_alarm_ignore_list(self):
+        """Step preserves a custom alarm_ignore_list through serialization."""
+
+        from nfv_vim.strategy.steps.kube_upgrade_steps import (
+            WaitKubernetesUpgradeHealthy,
+        )
+
+        custom_alarms = ["100.001", "200.002"]
+        step = WaitKubernetesUpgradeHealthy(
+            timeout_in_secs=180, alarm_ignore_list=custom_alarms
+        )
+
+        data = step.as_dict()
+        self.assertEqual(data["alarm_ignore_list"], custom_alarms)
+
+        new_step = object.__new__(WaitKubernetesUpgradeHealthy)
+        new_step.from_dict(data)
+        self.assertEqual(new_step._alarm_ignore_list, custom_alarms)
+
+    def test_from_dict_recreates_timer_on_resume(self):
+        """from_dict recreates the polling timer when step was in WAIT state."""
+
+        from nfv_common import timers
+        from nfv_vim.strategy.steps.kube_upgrade_steps import (
+            WaitKubernetesUpgradeHealthy,
+        )
+
+        # Simulate a step that was in progress (WAIT state) before restart
+        data = self.step.as_dict()
+        data["result"] = "wait"
+
+        new_step = object.__new__(WaitKubernetesUpgradeHealthy)
+        with mock.patch.object(
+            timers, "timers_create_timer", return_value=888
+        ) as mock_create:
+            new_step.from_dict(data)
+
+        mock_create.assert_called_once()
+        self.assertEqual(new_step._poll_timer_id, 888)
+        self.assertFalse(new_step._poll_in_progress)
+
+    def test_timeout_cleans_up_timer(self):
+        """Timeout cleans up the timer and provides a meaningful error."""
+
+        from nfv_common import timers
+
+        with mock.patch.object(timers, "timers_create_timer", return_value=999):
+            self.step.apply()
+
+        with mock.patch.object(timers, "timers_delete_timer") as mock_delete:
+            result, reason = self.step.timeout()
+
+        self.assertEqual(result, common_strategy.STRATEGY_STEP_RESULT.TIMED_OUT)
+        self.assertIn("did not become healthy before timeout", reason)
+        mock_delete.assert_called_once_with(999)
+        self.assertIsNone(self.step._poll_timer_id)
+
+    def test_flow_healthy_after_poll(self):
+        """Full flow: apply → timer polls → healthy → SUCCESS.
+
+        Simulates the flow where the step is applied, the timer fires,
+        issues a health query, and the callback reports the upgrade is
+        healthy.
+        """
+
+        from nfv_common import timers
+
+        # Step is applied and enters the WAIT state
+        with mock.patch.object(timers, "timers_create_timer", return_value=999):
+            result, reason = self.step.apply()
+        self.assertEqual(result, common_strategy.STRATEGY_STEP_RESULT.WAIT)
+
+        # Timer fires → _poll_action triggers query
+        with mock.patch("nfv_vim.nfvi.nfvi_get_kube_upgrade_health") as mock_query:
+            self.step._poll_action()
+            mock_query.assert_called_once()
+
+            # Drive the callback coroutine
+            callback = mock_query.call_args[0][1]
+            with mock.patch.object(timers, "timers_delete_timer"):
+                try:
+                    callback.send(
+                        {
+                            "completed": True,
+                            "result-data": "OK: all checks passed",
+                        }
+                    )
+                except StopIteration:
+                    pass
+
+        self.mock_stage.step_complete.assert_called_once_with(
+            common_strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
+        )
+
+    def test_flow_unhealthy_leads_to_timeout(self):
+        """Full flow: health check fails on every poll → eventually times out."""
+
+        from nfv_common import timers
+
+        # Step is applied and enters the WAIT state
+        with mock.patch.object(timers, "timers_create_timer", return_value=999):
+            result, reason = self.step.apply()
+        self.assertEqual(result, common_strategy.STRATEGY_STEP_RESULT.WAIT)
+
+        # Simulate multiple polls that all report unhealthy
+        for _ in range(3):
+            with mock.patch("nfv_vim.nfvi.nfvi_get_kube_upgrade_health") as mock_query:
+                self.step._poll_action()
+                mock_query.assert_called_once()
+
+                callback = mock_query.call_args[0][1]
+                try:
+                    callback.send(
+                        {
+                            "completed": True,
+                            "result-data": "[Fail] Kubernetes nodes not ready",
+                        }
+                    )
+                except StopIteration:
+                    pass
+
+            # Step should NOT have completed - still waiting
+            self.mock_stage.step_complete.assert_not_called()
+
+        # The strategy framework calls timeout() when time exceeds timeout_in_secs
+        with mock.patch.object(timers, "timers_delete_timer"):
+            result, reason = self.step.timeout()
+        self.assertEqual(result, common_strategy.STRATEGY_STEP_RESULT.TIMED_OUT)
+        self.assertEqual(
+            reason, "Kubernetes upgrade did not become healthy before timeout"
+        )
+
+    def test_flow_api_exception_retries_then_succeeds(self):
+        """Full flow: API exception causes retry, next poll succeeds."""
+
+        from nfv_common import timers
+
+        # Step is applied and enters the WAIT state
+        with mock.patch.object(timers, "timers_create_timer", return_value=999):
+            result, reason = self.step.apply()
+        self.assertEqual(result, common_strategy.STRATEGY_STEP_RESULT.WAIT)
+
+        # First poll - API exception occurs
+        with mock.patch("nfv_vim.nfvi.nfvi_get_kube_upgrade_health") as mock_query:
+            self.step._poll_action()
+            mock_query.assert_called_once()
+
+            callback = mock_query.call_args[0][1]
+            try:
+                callback.send({"completed": False, "reason": ""})
+            except StopIteration:
+                pass
+
+        # Step should NOT have failed - it retries
+        self.mock_stage.step_complete.assert_not_called()
+        self.assertFalse(self.step._poll_in_progress)
+
+        # Second poll - upgrade is healthy
+        with mock.patch("nfv_vim.nfvi.nfvi_get_kube_upgrade_health") as mock_query:
+            self.step._poll_action()
+            mock_query.assert_called_once()
+
+            callback = mock_query.call_args[0][1]
+            with mock.patch.object(timers, "timers_delete_timer"):
+                try:
+                    callback.send(
+                        {
+                            "completed": True,
+                            "result-data": "OK: all checks passed",
+                        }
+                    )
+                except StopIteration:
+                    pass
+
+        self.mock_stage.step_complete.assert_called_once_with(
+            common_strategy.STRATEGY_STEP_RESULT.SUCCESS, ""
+        )

nfv/nfv-vim/nfv_vim/nfvi/__init__.py

@@ -163,6 +163,9 @@ from nfv_vim.nfvi._nfvi_infrastructure_module import (  # noqa: F401
 from nfv_vim.nfvi._nfvi_infrastructure_module import (  # noqa: F401
     nfvi_get_kube_control_plane_pods_ready,
 )
+from nfv_vim.nfvi._nfvi_infrastructure_module import (  # noqa: F401
+    nfvi_get_kube_upgrade_health,
+)
 from nfv_vim.nfvi._nfvi_infrastructure_module import (  # noqa: F401
     nfvi_host_device_image_update,
 )

nfv/nfv-vim/nfv_vim/nfvi/_nfvi_infrastructure_module.py

@@ -672,6 +672,17 @@ def nfvi_get_kube_control_plane_pods_ready(callback):
     return cmd_id
 
 
+def nfvi_get_kube_upgrade_health(alarm_ignore_list, callback):
+    """Get kube upgrade health status."""
+
+    cmd_id = _infrastructure_plugin.invoke_plugin(
+        "get_kube_upgrade_health",
+        alarm_ignore_list,
+        callback=callback,
+    )
+    return cmd_id
+
+
 def nfvi_register_host_add_callback(callback):
     """Register for host add notifications."""
 

nfv/nfv-vim/nfv_vim/strategy/__init__.py

@@ -42,6 +42,7 @@ from nfv_vim.strategy.steps.kube_upgrade_steps import (  # noqa: F401
     QueryKubeUpgradeStep,
     QueryKubeVersionsStep,
     WaitKubeControlPlanePodsReadyStep,
+    WaitKubernetesUpgradeHealthy,
 )
 from nfv_vim.strategy._strategy_steps import DisableHostServicesStep  # noqa: F401
 from nfv_vim.strategy._strategy_steps import FwUpdateAbortHostsStep  # noqa: F401

nfv/nfv-vim/nfv_vim/strategy/_constants.py

@@ -62,6 +62,7 @@ class StrategyStepNames(Constants, metaclass=Singleton):
     QUERY_KUBE_ROOTCA_HOST_UPDATES = Constant("query-kube-rootca-host-updates")
     # kube upgrade steps
     WAIT_KUBE_CONTROL_PLANE_PODS_READY = Constant("wait-kube-control-plane-pods-ready")
+    KUBE_WAIT_UPGRADE_HEALTHY = Constant("kube-wait-upgrade-healthy")
     APPLY_PATCHES = Constant("apply-patches")
     QUERY_KUBE_HOST_UPGRADE = Constant("query-kube-host-upgrade")
     QUERY_KUBE_UPGRADE = Constant("query-kube-upgrade")

nfv/nfv-vim/nfv_vim/strategy/_strategy.py

@@ -2135,7 +2135,7 @@ class SwUpgradeStrategy(
 
             # Duplex: full kube upgrade runs sequentially after sw-deploy
             if self._kube_upgrade_version and is_duplex:
-                self._add_wait_kube_control_plane_pods_ready_stage()
+                self._add_wait_kubernetes_upgrade_healthy_stage()
                 self._build_kube_upgrade_stages()
                 if self._state == strategy.STRATEGY_STATE.BUILD_FAILED:
                     return

nfv/nfv-vim/nfv_vim/strategy/_strategy_stages.py

@@ -58,6 +58,7 @@ class StrategyStageNames(Constants, metaclass=Singleton):
     KUBE_ROOTCA_UPDATE_START = Constant("kube-rootca-update-start")
     # kube upgrade stages
     KUBE_WAIT_CONTROL_PLANE_PODS_READY = Constant("kube-wait-control-plane-pods-ready")
+    KUBE_WAIT_UPGRADE_HEALTHY = Constant("kube-wait-upgrade-healthy")
     KUBE_HOST_CORDON = Constant("kube-host-cordon")
     KUBE_HOST_UNCORDON = Constant("kube-host-uncordon")
     KUBE_UPGRADE_QUERY = Constant("kube-upgrade-query")

nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py

@@ -4735,6 +4735,9 @@ def strategy_step_rebuild_from_dict(data):
         STRATEGY_STEP_NAME.WAIT_KUBE_CONTROL_PLANE_PODS_READY: (
             kube_upgrade_steps.WaitKubeControlPlanePodsReadyStep
         ),
+        STRATEGY_STEP_NAME.KUBE_WAIT_UPGRADE_HEALTHY: (
+            kube_upgrade_steps.WaitKubernetesUpgradeHealthy
+        ),
         #
         # system config update steps
         #

nfv/nfv-vim/nfv_vim/strategy/stages/_kube_upgrade_stages.py

@@ -245,6 +245,17 @@ class KubeUpgradeStages:  # pylint: disable=no-member
         stage.add_step(strategy.WaitKubeControlPlanePodsReadyStep())
         self.apply_phase.add_stage(stage)
 
+    def _add_wait_kubernetes_upgrade_healthy_stage(self):
+        """Add wait for kubernetes upgrade healthy stage."""
+
+        from nfv_vim import strategy
+
+        stage = strategy.StrategyStage(
+            strategy.STRATEGY_STAGE_NAME.KUBE_WAIT_UPGRADE_HEALTHY
+        )
+        stage.add_step(strategy.WaitKubernetesUpgradeHealthy())
+        self.apply_phase.add_stage(stage)
+
     def _add_kube_upgrade_start_stage(self):
         """Add upgrade start strategy stage
 

nfv/nfv-vim/nfv_vim/strategy/steps/kube_upgrade_steps.py

@@ -1359,3 +1359,92 @@ class WaitKubeControlPlanePodsReadyStep(TimerBasedPollingStep):
         result, _ = super().timeout()
         reason = "Kubernetes control-plane pods did not become ready before timeout"
         return result, reason
+
+
+class WaitKubernetesUpgradeHealthy(TimerBasedPollingStep):
+    """Wait for Kubernetes Upgrade Healthy - Strategy Step.
+
+    Polls sysinv /health/kube-upgrade endpoint until no failures are
+    reported. Succeeds when the health response contains no '[Fail]'
+    entries.
+
+    Uses a self-scheduling timer to poll at a fixed interval, bypassing the
+    host audit mechanism which is gated by a 30-second delay.
+    """
+
+    POLL_INTERVAL_IN_SECS = 5
+    FIRST_POLL_DELAY_IN_SECS = 15
+
+    def __init__(self, timeout_in_secs=180, alarm_ignore_list=None):
+        super().__init__(
+            STRATEGY_STEP_NAME.KUBE_WAIT_UPGRADE_HEALTHY,
+            timeout_in_secs=timeout_in_secs,
+        )
+        if alarm_ignore_list is None:
+            alarm_ignore_list = KUBE_UPGRADE_START_ALARM_IGNORE
+        self._alarm_ignore_list = alarm_ignore_list
+
+    def _poll_action(self):
+        """Query kube upgrade health."""
+
+        from nfv_vim import nfvi
+
+        self._poll_in_progress = True
+        nfvi.nfvi_get_kube_upgrade_health(
+            self._alarm_ignore_list, self._query_health_callback()
+        )
+
+    @coroutine
+    def _query_health_callback(self):
+        """Query Kube Upgrade Health Callback."""
+
+        response = yield
+        DLOG.debug("Query-Kube-Upgrade-Health callback response=%s." % response)
+
+        self._poll_in_progress = False
+
+        if response["completed"]:
+            result_data = response.get("result-data")
+            if result_data is None:
+                self._cleanup_timer()
+                result = strategy.STRATEGY_STEP_RESULT.FAILED
+                self.stage.step_complete(
+                    result, "Kube upgrade health check missing result-data"
+                )
+                return
+
+            health_str = str(result_data)
+            if "[Fail]" in health_str:
+                DLOG.info("Kubernetes upgrade health check not passed: %s" % health_str)
+                # Keep waiting - timer will trigger the next query
+            else:
+                DLOG.info("Kubernetes upgrade health check passed: %s" % health_str)
+                self._cleanup_timer()
+                result = strategy.STRATEGY_STEP_RESULT.SUCCESS
+                self.stage.step_complete(result, "")
+        else:
+            # API call itself failed - don't fail the step, just retry
+            DLOG.warn("Query for kube upgrade health did not complete, will retry.")
+
+    def from_dict(self, data):
+        """Returns the step object initialized using the given dictionary."""
+
+        super().from_dict(data)
+        self._alarm_ignore_list = data.get(
+            "alarm_ignore_list", KUBE_UPGRADE_START_ALARM_IGNORE
+        )
+        return self
+
+    def as_dict(self):
+        """Represent the step as a dictionary."""
+
+        data = super().as_dict()
+        data["alarm_ignore_list"] = self._alarm_ignore_list
+        return data
+
+    def timeout(self):
+        """Strategy Step Timeout Override."""
+
+        result, _ = super().timeout()
+        reason = "Kubernetes upgrade did not become healthy before timeout"
+        return result, reason