Add new application update stages for k8s upgrades

Change-Id: I5217db1cb744320979afcd161cb436bf089ccc9d
Signed-off-by: Igor Soares <Igor.PiresSoares@windriver.com>

nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py

@@ -2016,6 +2016,124 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
             callback.send(response)
             callback.close()
 
+    def kube_pre_application_update(self, future, callback):
+        """
+        Start kube pre application update
+        """
+        response = dict()
+        response['completed'] = False
+        response['reason'] = ''
+        action_type = 'kube-pre-application-update'
+
+        try:
+            future.set_timeouts(config.CONF.get('nfvi-timeouts', None))
+
+            if self._platform_token is None or \
+                    self._platform_token.is_expired():
+                future.work(openstack.get_token, self._platform_directory)
+                future.result = (yield)
+
+                if not future.result.is_complete() or \
+                        future.result.data is None:
+                    DLOG.error("OpenStack get-token did not complete.")
+                    return
+
+                self._platform_token = future.result.data
+
+            future.work(sysinv.kube_pre_application_update,
+                        self._platform_token)
+            future.result = (yield)
+
+            if not future.result.is_complete():
+                DLOG.error("%s did not complete." % action_type)
+                return
+
+            kube_upgrade_data = future.result.data
+            kube_upgrade_obj = nfvi.objects.v1.KubeUpgrade(
+                kube_upgrade_data['state'],
+                kube_upgrade_data['from_version'],
+                kube_upgrade_data['to_version'])
+
+            response['result-data'] = kube_upgrade_obj
+            response['completed'] = True
+
+        except exceptions.OpenStackRestAPIException as e:
+            if httplib.UNAUTHORIZED == e.http_status_code:
+                response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED
+                if self._platform_token is not None:
+                    self._platform_token.set_expired()
+
+            else:
+                DLOG.exception("Caught API exception while trying %s. error=%s"
+                               % (action_type, e))
+
+        except Exception as e:
+            DLOG.exception("Caught exception while trying %s. error=%s"
+                           % (action_type, e))
+
+        finally:
+            callback.send(response)
+            callback.close()
+
+    def kube_post_application_update(self, future, callback):
+        """
+        Start kube post application update
+        """
+        response = dict()
+        response['completed'] = False
+        response['reason'] = ''
+        action_type = 'kube-post-application-update'
+
+        try:
+            future.set_timeouts(config.CONF.get('nfvi-timeouts', None))
+
+            if self._platform_token is None or \
+                    self._platform_token.is_expired():
+                future.work(openstack.get_token, self._platform_directory)
+                future.result = (yield)
+
+                if not future.result.is_complete() or \
+                        future.result.data is None:
+                    DLOG.error("OpenStack get-token did not complete.")
+                    return
+
+                self._platform_token = future.result.data
+
+            future.work(sysinv.kube_post_application_update,
+                        self._platform_token)
+            future.result = (yield)
+
+            if not future.result.is_complete():
+                DLOG.error("%s did not complete." % action_type)
+                return
+
+            kube_upgrade_data = future.result.data
+            kube_upgrade_obj = nfvi.objects.v1.KubeUpgrade(
+                kube_upgrade_data['state'],
+                kube_upgrade_data['from_version'],
+                kube_upgrade_data['to_version'])
+
+            response['result-data'] = kube_upgrade_obj
+            response['completed'] = True
+
+        except exceptions.OpenStackRestAPIException as e:
+            if httplib.UNAUTHORIZED == e.http_status_code:
+                response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED
+                if self._platform_token is not None:
+                    self._platform_token.set_expired()
+
+            else:
+                DLOG.exception("Caught API exception while trying %s. error=%s"
+                               % (action_type, e))
+
+        except Exception as e:
+            DLOG.exception("Caught exception while trying %s. error=%s"
+                           % (action_type, e))
+
+        finally:
+            callback.send(response)
+            callback.close()
+
     def kube_upgrade_networking(self, future, callback):
         """
         Start kube upgrade networking

nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/sysinv.py

@@ -436,6 +436,13 @@ def kube_upgrade_download_images(token):
     return _patch_kube_upgrade_state(token, "downloading-images")
 
 
+def kube_pre_application_update(token):
+    """
+    Ask System Inventory to kube pre application update
+    """
+    return _patch_kube_upgrade_state(token, "pre-updating-apps")
+
+
 def kube_upgrade_networking(token):
     """
     Ask System Inventory to kube upgrade networking
@@ -583,6 +590,13 @@ def upgrade_activate(token):
     return response
 
 
+def kube_post_application_update(token):
+    """
+    Ask System Inventory to kube post application update
+    """
+    return _patch_kube_upgrade_state(token, "post-updating-apps")
+
+
 def upgrade_complete(token):
     """
     Ask System Inventory to complete an upgrade

nfv/nfv-tests/nfv_unit_tests/tests/test_kube_upgrade_strategy.py

@@ -269,8 +269,7 @@ class ApplyStageMixin(object):
             'total_steps': 1,
             'steps': [
                 {'name': 'kube-upgrade-start',
-                 'success_state': 'upgrade-started',
+                 'success_state': 'upgrade-started'},
-                 'fail_state': 'upgrade-starting-failed'},
             ],
         }
 
@@ -285,6 +284,17 @@ class ApplyStageMixin(object):
             ],
         }
 
+    def _kube_pre_application_update_stage(self):
+        return {
+            'name': 'kube-pre-application-update',
+            'total_steps': 1,
+            'steps': [
+                {'name': 'kube-pre-application-update',
+                 'success_state': 'pre-updated-apps',
+                 'fail_state': 'pre-updating-apps-failed'},
+            ],
+        }
+
     def _kube_host_cordon_stage(self, ver="N/A"):
         return {
             'name': 'kube-host-cordon',
@@ -352,6 +362,17 @@ class ApplyStageMixin(object):
             ],
         }
 
+    def _kube_post_application_update_stage(self):
+        return {
+            'name': 'kube-post-application-update',
+            'total_steps': 1,
+            'steps': [
+                {'name': 'kube-post-application-update',
+                 'success_state': 'post-updated-apps',
+                 'fail_state': 'post-updating-apps-failed'},
+            ],
+        }
+
     def _kube_upgrade_complete_stage(self):
         return {
             'name': 'kube-upgrade-complete',
@@ -496,6 +517,7 @@ class ApplyStageMixin(object):
                          storage_list=None,
                          add_start=True,
                          add_download=True,
+                         add_pre_app_update=True,
                          add_networking=True,
                          add_storage=True,
                          add_cordon=True,
@@ -503,6 +525,7 @@ class ApplyStageMixin(object):
                          add_second_control_plane=True,
                          add_kubelets=True,
                          add_uncordon=True,
+                         add_post_app_update=True,
                          add_complete=True,
                          add_cleanup=True):
         """The order of the host_list determines the kubelets"""
@@ -518,6 +541,8 @@ class ApplyStageMixin(object):
             stages.append(self._kube_upgrade_start_stage())
         if add_download:
             stages.append(self._kube_upgrade_download_images_stage())
+        if add_pre_app_update:
+            stages.append(self._kube_pre_application_update_stage())
         if add_networking:
             stages.append(self._kube_upgrade_networking_stage())
         if add_storage:
@@ -537,6 +562,8 @@ class ApplyStageMixin(object):
                                                                 worker_list))
         if add_uncordon:
             stages.append(self._kube_host_uncordon_stage())
+        if add_post_app_update:
+            stages.append(self._kube_post_application_update_stage())
         if add_complete:
             stages.append(self._kube_upgrade_complete_stage())
         if add_cleanup:
@@ -624,39 +651,6 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
         self.worker_list = []
         self.storage_list = []
 
-    def test_resume_after_starting_failed(self):
-        """
-        Test the kube_upgrade strategy creation when the upgrade had previously
-        stopped with 'upgrade-starting-failed'
-        It is expected to resume at the 'upgrade-starting' stage
-        """
-        kube_upgrade = self._create_kube_upgrade_obj(
-            KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTING_FAILED,
-            self.default_from_version,
-            self.default_to_version)
-        stages = [
-            self._kube_upgrade_start_stage(),
-            self._kube_upgrade_download_images_stage(),
-            self._kube_upgrade_networking_stage(),
-            self._kube_upgrade_storage_stage(),
-        ]
-        if self.is_simplex():
-            stages.append(self._kube_host_cordon_stage())
-        for ver in self.kube_versions:
-            stages.append(self._kube_upgrade_first_control_plane_stage(ver))
-            stages.extend(self._kube_upgrade_kubelet_stages(
-                ver,
-                self.std_controller_list,
-                self.aio_controller_list,
-                self.worker_list))
-        if self.is_simplex():
-            stages.append(self._kube_host_uncordon_stage())
-        stages.extend([
-            self._kube_upgrade_complete_stage(),
-            self._kube_upgrade_cleanup_stage(),
-        ])
-        self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
-
     def test_resume_after_download_images_failed(self):
         """
         Test the kube_upgrade strategy creation when the upgrade had previously
@@ -669,6 +663,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
             self.default_to_version)
         stages = [
             self._kube_upgrade_download_images_stage(),
+            self._kube_pre_application_update_stage(),
             self._kube_upgrade_networking_stage(),
             self._kube_upgrade_storage_stage(),
         ]
@@ -684,6 +679,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
         if self.is_simplex():
             stages.append(self._kube_host_uncordon_stage())
         stages.extend([
+            self._kube_post_application_update_stage(),
             self._kube_upgrade_complete_stage(),
             self._kube_upgrade_cleanup_stage(),
         ])
@@ -693,12 +689,79 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
         """
         Test the kube_upgrade strategy creation when the upgrade had previously
         stopped with 'downloaded-images'
-        It is expected to resume at the 'first control plane' stage.
+        It is expected to resume at the 'pre application update' stage.
         """
         kube_upgrade = self._create_kube_upgrade_obj(
             KUBE_UPGRADE_STATE.KUBE_UPGRADE_DOWNLOADED_IMAGES,
             self.default_from_version,
             self.default_to_version)
+        stages = [
+            self._kube_pre_application_update_stage(),
+            self._kube_upgrade_networking_stage(),
+            self._kube_upgrade_storage_stage(),
+        ]
+        if self.is_simplex():
+            stages.append(self._kube_host_cordon_stage())
+        for ver in self.kube_versions:
+            stages.append(self._kube_upgrade_first_control_plane_stage(
+                ver))
+            stages.extend(self._kube_upgrade_kubelet_stages(
+                ver,
+                self.std_controller_list,
+                self.aio_controller_list,
+                self.worker_list))
+        if self.is_simplex():
+            stages.append(self._kube_host_uncordon_stage())
+        stages.extend([
+            self._kube_post_application_update_stage(),
+            self._kube_upgrade_complete_stage(),
+            self._kube_upgrade_cleanup_stage(),
+        ])
+        self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
+
+    def test_resume_after_pre_app_update_failed(self):
+        """
+        Test the kube_upgrade strategy creation when the upgrade had previously
+        stopped with 'pre-updating-apps-failed'.
+        It is expected to resume at the pre application update stage.
+        """
+        kube_upgrade = self._create_kube_upgrade_obj(
+            KUBE_UPGRADE_STATE.KUBE_PRE_UPDATING_APPS_FAILED,
+            self.default_from_version,
+            self.default_to_version)
+        stages = [
+            self._kube_pre_application_update_stage(),
+            self._kube_upgrade_networking_stage(),
+            self._kube_upgrade_storage_stage(),
+        ]
+        if self.is_simplex():
+            stages.append(self._kube_host_cordon_stage())
+        for ver in self.kube_versions:
+            stages.append(self._kube_upgrade_first_control_plane_stage(ver))
+            stages.extend(self._kube_upgrade_kubelet_stages(
+                ver,
+                self.std_controller_list,
+                self.aio_controller_list,
+                self.worker_list))
+        if self.is_simplex():
+            stages.append(self._kube_host_uncordon_stage())
+        stages.extend([
+            self._kube_post_application_update_stage(),
+            self._kube_upgrade_complete_stage(),
+            self._kube_upgrade_cleanup_stage(),
+        ])
+        self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
+
+    def test_resume_after_pre_app_update_succeeded(self):
+        """
+        Test the kube_upgrade strategy creation when the upgrade had previously
+        stopped with 'pre-updated-apps'.
+        It is expected to resume at the upgrade networking stage.
+        """
+        kube_upgrade = self._create_kube_upgrade_obj(
+            KUBE_UPGRADE_STATE.KUBE_PRE_UPDATED_APPS,
+            self.default_from_version,
+            self.default_to_version)
         stages = [
             self._kube_upgrade_networking_stage(),
             self._kube_upgrade_storage_stage(),
@@ -716,6 +779,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
         if self.is_simplex():
             stages.append(self._kube_host_uncordon_stage())
         stages.extend([
+            self._kube_post_application_update_stage(),
             self._kube_upgrade_complete_stage(),
             self._kube_upgrade_cleanup_stage(),
         ])
@@ -743,6 +807,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
         if self.is_simplex():
             stages.append(self._kube_host_uncordon_stage())
         stages.extend([
+            self._kube_post_application_update_stage(),
             self._kube_upgrade_complete_stage(),
             self._kube_upgrade_cleanup_stage(),
         ])
@@ -768,6 +833,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
         if self.is_simplex():
             stages.append(self._kube_host_uncordon_stage())
         stages.extend([
+            self._kube_post_application_update_stage(),
             self._kube_upgrade_complete_stage(),
             self._kube_upgrade_cleanup_stage(),
         ])
@@ -800,6 +866,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
         if self.is_simplex():
             stages.append(self._kube_host_uncordon_stage())
         stages.extend([
+            self._kube_post_application_update_stage(),
             self._kube_upgrade_complete_stage(),
             self._kube_upgrade_cleanup_stage(),
         ])
@@ -831,6 +898,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
         if self.is_simplex():
             stages.append(self._kube_host_uncordon_stage())
         stages.extend([
+            self._kube_post_application_update_stage(),
             self._kube_upgrade_complete_stage(),
             self._kube_upgrade_cleanup_stage(),
         ])
@@ -862,6 +930,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
         if self.is_simplex():
             stages.append(self._kube_host_uncordon_stage())
         stages.extend([
+            self._kube_post_application_update_stage(),
             self._kube_upgrade_complete_stage(),
             self._kube_upgrade_cleanup_stage(),
         ])
@@ -891,6 +960,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
         if self.is_simplex():
             stages.append(self._kube_host_uncordon_stage())
         stages.extend([
+            self._kube_post_application_update_stage(),
             self._kube_upgrade_complete_stage(),
             self._kube_upgrade_cleanup_stage(),
         ])
@@ -918,6 +988,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
         if self.is_simplex():
             stages.append(self._kube_host_uncordon_stage())
         stages.extend([
+            self._kube_post_application_update_stage(),
             self._kube_upgrade_complete_stage(),
             self._kube_upgrade_cleanup_stage(),
         ])
@@ -945,11 +1016,45 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
         if self.is_simplex():
             stages.append(self._kube_host_uncordon_stage())
         stages.extend([
+            self._kube_post_application_update_stage(),
             self._kube_upgrade_complete_stage(),
             self._kube_upgrade_cleanup_stage(),
         ])
         self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
 
+    def test_resume_after_post_app_update_failed(self):
+        """
+        Test the kube_upgrade strategy creation when the upgrade had previously
+        stopped with 'post-updating-apps-failed'
+        It is expected to resume at the post application update stage.
+        """
+        kube_upgrade = self._create_kube_upgrade_obj(
+            KUBE_UPGRADE_STATE.KUBE_POST_UPDATING_APPS_FAILED,
+            self.default_from_version,
+            self.default_to_version)
+        stages = [
+            self._kube_post_application_update_stage(),
+            self._kube_upgrade_complete_stage(),
+            self._kube_upgrade_cleanup_stage(),
+        ]
+        self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
+
+    def test_resume_after_post_app_update_succeeded(self):
+        """
+        Test the kube_upgrade strategy creation when the upgrade had previously
+        stopped with 'post-updated-apps'.
+        It is expected to resume at the upgrade complete stage.
+        """
+        kube_upgrade = self._create_kube_upgrade_obj(
+            KUBE_UPGRADE_STATE.KUBE_POST_UPDATED_APPS,
+            self.default_from_version,
+            self.default_to_version)
+        stages = [
+            self._kube_upgrade_complete_stage(),
+            self._kube_upgrade_cleanup_stage(),
+        ]
+        self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
+
 
 @mock.patch('nfv_vim.event_log._instance._event_issue',
             sw_update_testcase.fake_event_issue)

nfv/nfv-vim/nfv_vim/nfvi/__init__.py

@@ -117,6 +117,8 @@ from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_host_cordon  # no
 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_host_uncordon  # noqa: F401
 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_host_upgrade_control_plane  # noqa: F401
 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_host_upgrade_kubelet  # noqa: F401
+from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_post_application_update  # noqa: F401
+from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_pre_application_update  # noqa: F401
 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_rootca_update_abort  # noqa: F401
 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_rootca_update_complete  # noqa: F401
 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_rootca_update_generate_cert  # noqa: F401

nfv/nfv-vim/nfv_vim/nfvi/_nfvi_infrastructure_module.py

@@ -321,6 +321,26 @@ def nfvi_kube_upgrade_start(to_version, force, alarm_ignore_list, callback):
     return cmd_id
 
 
+def nfvi_kube_pre_application_update(callback):
+    """
+    Kube Upgrade Pre Application Update
+    """
+    cmd_id = _infrastructure_plugin.invoke_plugin(
+        'kube_pre_application_update',
+        callback=callback)
+    return cmd_id
+
+
+def nfvi_kube_post_application_update(callback):
+    """
+    Kube Upgrade Post Application Update
+    """
+    cmd_id = _infrastructure_plugin.invoke_plugin(
+        'kube_post_application_update',
+        callback=callback)
+    return cmd_id
+
+
 def nfvi_get_kube_host_upgrade_list(callback):
     """
     Get kube host upgrade list

nfv/nfv-vim/nfv_vim/nfvi/objects/v1/_kube_upgrade.py

@@ -37,12 +37,16 @@ class KubeUpgradeState(Constants):
     Maintaining the same order as defined in kubernetes.py
     """
 
-    KUBE_UPGRADE_STARTING = Constant('upgrade-starting')
-    KUBE_UPGRADE_STARTING_FAILED = Constant('upgrade-starting-failed')
     KUBE_UPGRADE_STARTED = Constant('upgrade-started')
     KUBE_UPGRADE_DOWNLOADING_IMAGES = Constant('downloading-images')
     KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED = Constant('downloading-images-failed')
     KUBE_UPGRADE_DOWNLOADED_IMAGES = Constant('downloaded-images')
+    KUBE_PRE_UPDATING_APPS = Constant('pre-updating-apps')
+    KUBE_PRE_UPDATING_APPS_FAILED = Constant('pre-updating-apps-failed')
+    KUBE_PRE_UPDATED_APPS = Constant('pre-updated-apps')
+    KUBE_POST_UPDATING_APPS = Constant('post-updating-apps')
+    KUBE_POST_UPDATING_APPS_FAILED = Constant('post-updating-apps-failed')
+    KUBE_POST_UPDATED_APPS = Constant('post-updated-apps')
     KUBE_UPGRADING_NETWORKING = Constant('upgrading-networking')
     KUBE_UPGRADING_NETWORKING_FAILED = Constant('upgrading-networking-failed')
     KUBE_UPGRADED_NETWORKING = Constant('upgraded-networking')

nfv/nfv-vim/nfv_vim/strategy/__init__.py

@@ -21,6 +21,8 @@ from nfv_vim.strategy._strategy_steps import KubeHostCordonStep  # noqa: F401
 from nfv_vim.strategy._strategy_steps import KubeHostUncordonStep  # noqa: F401
 from nfv_vim.strategy._strategy_steps import KubeHostUpgradeControlPlaneStep  # noqa: F401
 from nfv_vim.strategy._strategy_steps import KubeHostUpgradeKubeletStep  # noqa: F401
+from nfv_vim.strategy._strategy_steps import KubePostApplicationUpdateStep  # noqa: F401
+from nfv_vim.strategy._strategy_steps import KubePreApplicationUpdateStep  # noqa: F401
 from nfv_vim.strategy._strategy_steps import KubeRootcaUpdateCompleteStep  # noqa: F401
 from nfv_vim.strategy._strategy_steps import KubeRootcaUpdateGenerateCertStep  # noqa: F401
 from nfv_vim.strategy._strategy_steps import KubeRootcaUpdateHostTrustBothcasStep  # noqa: F401

nfv/nfv-vim/nfv_vim/strategy/_strategy.py

@@ -3454,7 +3454,23 @@ class KubeUpgradeStrategy(SwUpdateStrategy,
             strategy.STRATEGY_STAGE_NAME.KUBE_UPGRADE_DOWNLOAD_IMAGES)
         stage.add_step(strategy.KubeUpgradeDownloadImagesStep())
         self.apply_phase.add_stage(stage)
-        # Next stage after download images is upgrade networking
+        # Next stage after download images is pre application update
+        self._add_kube_pre_application_update_stage()
+
+    def _add_kube_pre_application_update_stage(self):
+        """
+        Add kube pre application update stage.
+        This stage only occurs after download images
+        It then proceeds to the next stage
+        """
+        from nfv_vim import strategy
+
+        stage = strategy.StrategyStage(
+            strategy.STRATEGY_STAGE_NAME.KUBE_PRE_APPLICATION_UPDATE)
+        stage.add_step(strategy.KubePreApplicationUpdateStep())
+        self.apply_phase.add_stage(stage)
+
+        # Next stage after pre application update is upgrade networking
         self._add_kube_upgrade_networking_stage()
 
     def _add_kube_upgrade_networking_stage(self):
@@ -3599,8 +3615,8 @@ class KubeUpgradeStrategy(SwUpdateStrategy,
                 nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_HOST_UNCORDON_FAILED)
             )
             self.apply_phase.add_stage(stage)
-        # after this loop is kube upgrade complete stage
+        # after this loop is kube post application update stage
-        self._add_kube_upgrade_complete_stage()
+        self._add_kube_post_application_update_stage()
 
     def _add_kube_upgrade_first_control_plane_stage(self, first_host, kube_ver):
         """Add first controller control plane kube upgrade stage"""
@@ -3729,10 +3745,26 @@ class KubeUpgradeStrategy(SwUpdateStrategy,
                     self.report_build_failure(reason)
                     return
 
+    def _add_kube_post_application_update_stage(self):
+        """
+        Add kube post application update stage.
+        This stage occurs after all kubelets are upgraded and hosts uncordoned
+        It then proceeds to the next stage
+        """
+        from nfv_vim import strategy
+
+        stage = strategy.StrategyStage(
+            strategy.STRATEGY_STAGE_NAME.KUBE_POST_APPLICATION_UPDATE)
+        stage.add_step(strategy.KubePostApplicationUpdateStep())
+        self.apply_phase.add_stage(stage)
+
+        # Next stage after post application update is upgrade complete
+        self._add_kube_upgrade_complete_stage()
+
     def _add_kube_upgrade_complete_stage(self):
         """
         Add kube upgrade complete strategy stage
-        This stage occurs after all kubelets are upgraded
+        This stage occurs after apps are post updated
         """
         from nfv_vim import strategy
         stage = strategy.StrategyStage(
@@ -3806,10 +3838,6 @@ class KubeUpgradeStrategy(SwUpdateStrategy,
         # Note: there are no resume states for actions that are still running
         # ie:  KUBE_UPGRADE_DOWNLOADING_IMAGES
         RESUME_STATE = {
-            # If upgrade start failed, allow to restart
-            nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTING_FAILED:
-                self._add_kube_upgrade_start_stage,
-
             # after upgrade-started -> download images
             nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTED:
                 self._add_kube_upgrade_download_images_stage,
@@ -3818,8 +3846,16 @@ class KubeUpgradeStrategy(SwUpdateStrategy,
             nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED:
                 self._add_kube_upgrade_download_images_stage,
 
-            # After downloading images -> upgrade networking
+            # After downloading images -> pre update applications
             nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_DOWNLOADED_IMAGES:
+                self._add_kube_pre_application_update_stage,
+
+            # if pre updating apps failed, resume at pre updating apps
+            nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_PRE_UPDATING_APPS_FAILED:
+                self._add_kube_pre_application_update_stage,
+
+            # After pre updating apps -> upgrade networking
+            nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_PRE_UPDATED_APPS:
                 self._add_kube_upgrade_networking_stage,
 
             # if networking state failed, resync at networking state
@@ -3870,8 +3906,16 @@ class KubeUpgradeStrategy(SwUpdateStrategy,
             nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_HOST_UNCORDON_FAILED:
                 self._add_kube_host_uncordon_stage,
 
-            # If the state is uncordon-complete, resume at complete stage
+            # After uncordon-complete -> post update apps
             nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_HOST_UNCORDON_COMPLETE:
+                self._add_kube_post_application_update_stage,
+
+            # If post updating apps failed, resume at post application update stage
+            nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_POST_UPDATING_APPS_FAILED:
+                self._add_kube_post_application_update_stage,
+
+            # After post updating apps -> upgrade complete
+            nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_POST_UPDATED_APPS:
                 self._add_kube_upgrade_complete_stage,
 
             # upgrade is completed, delete the upgrade

nfv/nfv-vim/nfv_vim/strategy/_strategy_stages.py

@@ -63,6 +63,7 @@ class StrategyStageNames(Constants):
     KUBE_UPGRADE_QUERY = Constant('kube-upgrade-query')
     KUBE_UPGRADE_START = Constant('kube-upgrade-start')
     KUBE_UPGRADE_DOWNLOAD_IMAGES = Constant('kube-upgrade-download-images')
+    KUBE_PRE_APPLICATION_UPDATE = Constant('kube-pre-application-update')
     KUBE_UPGRADE_FIRST_CONTROL_PLANE = \
         Constant('kube-upgrade-first-control-plane')
     KUBE_UPGRADE_NETWORKING = Constant('kube-upgrade-networking')
@@ -73,6 +74,7 @@ class StrategyStageNames(Constants):
     KUBE_UPGRADE_KUBELETS_CONTROLLERS = \
        Constant('kube-upgrade-kubelets-controllers')
     KUBE_UPGRADE_KUBELETS_WORKERS = Constant('kube-upgrade-kubelets-workers')
+    KUBE_POST_APPLICATION_UPDATE = Constant('kube-post-application-update')
     KUBE_UPGRADE_COMPLETE = Constant('kube-upgrade-complete')
     KUBE_UPGRADE_CLEANUP = Constant('kube-upgrade-cleanup')
 

nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py

@@ -87,6 +87,8 @@ class StrategyStepNames(Constants):
     KUBE_HOST_UPGRADE_CONTROL_PLANE = \
         Constant('kube-host-upgrade-control-plane')
     KUBE_HOST_UPGRADE_KUBELET = Constant('kube-host-upgrade-kubelet')
+    KUBE_PRE_APPLICATION_UPDATE = Constant('kube-pre-application-update')
+    KUBE_POST_APPLICATION_UPDATE = Constant('kube-post-application-update')
     # system config update specific steps
     QUERY_SYSTEM_CONFIG_UPDATE_HOSTS = Constant('query-system-config-update-hosts')
     SYSTEM_CONFIG_UPDATE_HOSTS = Constant('system-config-update-hosts')
@@ -4172,8 +4174,7 @@ class KubeUpgradeStartStep(AbstractKubeUpgradeStep):
         super(KubeUpgradeStartStep, self).__init__(
             STRATEGY_STEP_NAME.KUBE_UPGRADE_START,
             nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTED,
-            nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTING_FAILED,
+            None)  # there is no failure state if upgrade-start fails
-            timeout_in_secs=1800)
         # next 2 attributes must be persisted through from_dict/as_dict
         self._to_version = to_version
         self._force = force
@@ -4209,9 +4210,13 @@ class KubeUpgradeStartStep(AbstractKubeUpgradeStep):
         response = (yield)
         DLOG.debug("%s callback response=%s." % (self._name, response))
 
+        # kube-upgrade-start will return a result when it completes,
+        # so we do not want to use handle_event
         if response['completed']:
             if self.strategy is not None:
                 self.strategy.nfvi_kube_upgrade = response['result-data']
+            result = strategy.STRATEGY_STEP_RESULT.SUCCESS
+            self.stage.step_complete(result, "")
         else:
             result = strategy.STRATEGY_STEP_RESULT.FAILED
             self.stage.step_complete(result, response['reason'])
@@ -4401,6 +4406,88 @@ class KubeUpgradeDownloadImagesStep(AbstractKubeUpgradeStep):
         return strategy.STRATEGY_STEP_RESULT.WAIT, ""
 
 
+class KubePreApplicationUpdateStep(AbstractKubeUpgradeStep):
+    """Kube Pre Application Update - Strategy Step"""
+
+    def __init__(self):
+        from nfv_vim import nfvi
+        super(KubePreApplicationUpdateStep, self).__init__(
+            STRATEGY_STEP_NAME.KUBE_PRE_APPLICATION_UPDATE,
+            nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_PRE_UPDATED_APPS,
+            nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_PRE_UPDATING_APPS_FAILED,
+            timeout_in_secs=1800)
+
+    def abort(self):
+        """
+        Returns the abort step related to this step
+        """
+        return [KubeUpgradeAbortStep()]
+
+    @coroutine
+    def _response_callback(self):
+        """Kube Pre Application Update - Callback"""
+
+        response = (yield)
+        DLOG.debug("%s callback response=%s." % (self._name, response))
+
+        if response['completed']:
+            if self.strategy is not None:
+                self.strategy.nfvi_kube_upgrade = response['result-data']
+        else:
+            result = strategy.STRATEGY_STEP_RESULT.FAILED
+            self.stage.step_complete(result, response['reason'])
+
+    def apply(self):
+        """Kube Pre Application Update """
+
+        from nfv_vim import nfvi
+        DLOG.info("Step (%s) apply." % self._name)
+
+        nfvi.nfvi_kube_pre_application_update(self._response_callback())
+        return strategy.STRATEGY_STEP_RESULT.WAIT, ""
+
+
+class KubePostApplicationUpdateStep(AbstractKubeUpgradeStep):
+    """Kube Post Application Update - Strategy Step"""
+
+    def __init__(self):
+        from nfv_vim import nfvi
+        super(KubePostApplicationUpdateStep, self).__init__(
+            STRATEGY_STEP_NAME.KUBE_POST_APPLICATION_UPDATE,
+            nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_POST_UPDATED_APPS,
+            nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_POST_UPDATING_APPS_FAILED,
+            timeout_in_secs=1800)
+
+    def abort(self):
+        """
+        Returns the abort step related to this step
+        """
+        return [KubeUpgradeAbortStep()]
+
+    @coroutine
+    def _response_callback(self):
+        """Kube Post Application Update - Callback"""
+
+        response = (yield)
+        DLOG.debug("%s callback response=%s." % (self._name, response))
+
+        if response['completed']:
+            if self.strategy is not None:
+                self.strategy.nfvi_kube_upgrade = response['result-data']
+        else:
+            result = strategy.STRATEGY_STEP_RESULT.FAILED
+            self.stage.step_complete(result, response['reason'])
+
+    def apply(self):
+        """Kube Post Application Update """
+
+        from nfv_vim import nfvi
+        DLOG.info("Step (%s) apply." % self._name)
+
+        nfvi.nfvi_kube_post_application_update(self._response_callback())
+        return strategy.STRATEGY_STEP_RESULT.WAIT, ""
+
+
 class KubeUpgradeNetworkingStep(AbstractKubeUpgradeStep):
     """Kube Upgrade Networking - Strategy Step"""
 
@@ -4932,6 +5019,8 @@ def strategy_step_rebuild_from_dict(data):
         STRATEGY_STEP_NAME.KUBE_UPGRADE_NETWORKING: KubeUpgradeNetworkingStep,
         STRATEGY_STEP_NAME.KUBE_UPGRADE_STORAGE: KubeUpgradeStorageStep,
         STRATEGY_STEP_NAME.KUBE_UPGRADE_START: KubeUpgradeStartStep,
+        STRATEGY_STEP_NAME.KUBE_PRE_APPLICATION_UPDATE: KubePreApplicationUpdateStep,
+        STRATEGY_STEP_NAME.KUBE_POST_APPLICATION_UPDATE: KubePostApplicationUpdateStep,
         STRATEGY_STEP_NAME.QUERY_KUBE_HOST_UPGRADE: QueryKubeHostUpgradeStep,
         STRATEGY_STEP_NAME.QUERY_KUBE_UPGRADE: QueryKubeUpgradeStep,
         STRATEGY_STEP_NAME.QUERY_KUBE_VERSIONS: QueryKubeVersionsStep,