From 4ddfea2c0f18389f6e6e7b3acee64e173bbc165c Mon Sep 17 00:00:00 2001 From: Igor Soares Date: Wed, 27 Mar 2024 19:16:15 -0300 Subject: [PATCH] Add new application update stages for k8s upgrades Change-Id: I5217db1cb744320979afcd161cb436bf089ccc9d Signed-off-by: Igor Soares --- .../nfvi_plugins/nfvi_infrastructure_api.py | 118 ++++++++++++++ .../nfvi_plugins/openstack/sysinv.py | 14 ++ .../tests/test_kube_upgrade_strategy.py | 144 +++++++++++++----- nfv/nfv-vim/nfv_vim/nfvi/__init__.py | 2 + .../nfvi/_nfvi_infrastructure_module.py | 20 +++ .../nfv_vim/nfvi/objects/v1/_kube_upgrade.py | 8 +- nfv/nfv-vim/nfv_vim/strategy/__init__.py | 2 + nfv/nfv-vim/nfv_vim/strategy/_strategy.py | 64 ++++++-- .../nfv_vim/strategy/_strategy_stages.py | 2 + .../nfv_vim/strategy/_strategy_steps.py | 93 ++++++++++- 10 files changed, 417 insertions(+), 50 deletions(-) diff --git a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py index 0c7ac08b..de5e8b35 100755 --- a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py +++ b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py @@ -2016,6 +2016,124 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI): callback.send(response) callback.close() + def kube_pre_application_update(self, future, callback): + """ + Start kube pre application update + """ + response = dict() + response['completed'] = False + response['reason'] = '' + action_type = 'kube-pre-application-update' + + try: + future.set_timeouts(config.CONF.get('nfvi-timeouts', None)) + + if self._platform_token is None or \ + self._platform_token.is_expired(): + future.work(openstack.get_token, self._platform_directory) + future.result = (yield) + + if not future.result.is_complete() or \ + future.result.data is None: + DLOG.error("OpenStack get-token did not complete.") + return + + self._platform_token = future.result.data + + future.work(sysinv.kube_pre_application_update, + self._platform_token) + future.result = (yield) + + if not future.result.is_complete(): + DLOG.error("%s did not complete." % action_type) + return + + kube_upgrade_data = future.result.data + kube_upgrade_obj = nfvi.objects.v1.KubeUpgrade( + kube_upgrade_data['state'], + kube_upgrade_data['from_version'], + kube_upgrade_data['to_version']) + + response['result-data'] = kube_upgrade_obj + response['completed'] = True + + except exceptions.OpenStackRestAPIException as e: + if httplib.UNAUTHORIZED == e.http_status_code: + response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED + if self._platform_token is not None: + self._platform_token.set_expired() + + else: + DLOG.exception("Caught API exception while trying %s. error=%s" + % (action_type, e)) + + except Exception as e: + DLOG.exception("Caught exception while trying %s. error=%s" + % (action_type, e)) + + finally: + callback.send(response) + callback.close() + + def kube_post_application_update(self, future, callback): + """ + Start kube post application update + """ + response = dict() + response['completed'] = False + response['reason'] = '' + action_type = 'kube-post-application-update' + + try: + future.set_timeouts(config.CONF.get('nfvi-timeouts', None)) + + if self._platform_token is None or \ + self._platform_token.is_expired(): + future.work(openstack.get_token, self._platform_directory) + future.result = (yield) + + if not future.result.is_complete() or \ + future.result.data is None: + DLOG.error("OpenStack get-token did not complete.") + return + + self._platform_token = future.result.data + + future.work(sysinv.kube_post_application_update, + self._platform_token) + future.result = (yield) + + if not future.result.is_complete(): + DLOG.error("%s did not complete." % action_type) + return + + kube_upgrade_data = future.result.data + kube_upgrade_obj = nfvi.objects.v1.KubeUpgrade( + kube_upgrade_data['state'], + kube_upgrade_data['from_version'], + kube_upgrade_data['to_version']) + + response['result-data'] = kube_upgrade_obj + response['completed'] = True + + except exceptions.OpenStackRestAPIException as e: + if httplib.UNAUTHORIZED == e.http_status_code: + response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED + if self._platform_token is not None: + self._platform_token.set_expired() + + else: + DLOG.exception("Caught API exception while trying %s. error=%s" + % (action_type, e)) + + except Exception as e: + DLOG.exception("Caught exception while trying %s. error=%s" + % (action_type, e)) + + finally: + callback.send(response) + callback.close() + def kube_upgrade_networking(self, future, callback): """ Start kube upgrade networking diff --git a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/sysinv.py b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/sysinv.py index 5fb4baff..129c5169 100755 --- a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/sysinv.py +++ b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/sysinv.py @@ -436,6 +436,13 @@ def kube_upgrade_download_images(token): return _patch_kube_upgrade_state(token, "downloading-images") +def kube_pre_application_update(token): + """ + Ask System Inventory to kube pre application update + """ + return _patch_kube_upgrade_state(token, "pre-updating-apps") + + def kube_upgrade_networking(token): """ Ask System Inventory to kube upgrade networking @@ -583,6 +590,13 @@ def upgrade_activate(token): return response +def kube_post_application_update(token): + """ + Ask System Inventory to kube post application update + """ + return _patch_kube_upgrade_state(token, "post-updating-apps") + + def upgrade_complete(token): """ Ask System Inventory to complete an upgrade diff --git a/nfv/nfv-tests/nfv_unit_tests/tests/test_kube_upgrade_strategy.py b/nfv/nfv-tests/nfv_unit_tests/tests/test_kube_upgrade_strategy.py index 5849f08f..6dfaad5d 100755 --- a/nfv/nfv-tests/nfv_unit_tests/tests/test_kube_upgrade_strategy.py +++ b/nfv/nfv-tests/nfv_unit_tests/tests/test_kube_upgrade_strategy.py @@ -269,8 +269,7 @@ class ApplyStageMixin(object): 'total_steps': 1, 'steps': [ {'name': 'kube-upgrade-start', - 'success_state': 'upgrade-started', - 'fail_state': 'upgrade-starting-failed'}, + 'success_state': 'upgrade-started'}, ], } @@ -285,6 +284,17 @@ class ApplyStageMixin(object): ], } + def _kube_pre_application_update_stage(self): + return { + 'name': 'kube-pre-application-update', + 'total_steps': 1, + 'steps': [ + {'name': 'kube-pre-application-update', + 'success_state': 'pre-updated-apps', + 'fail_state': 'pre-updating-apps-failed'}, + ], + } + def _kube_host_cordon_stage(self, ver="N/A"): return { 'name': 'kube-host-cordon', @@ -352,6 +362,17 @@ class ApplyStageMixin(object): ], } + def _kube_post_application_update_stage(self): + return { + 'name': 'kube-post-application-update', + 'total_steps': 1, + 'steps': [ + {'name': 'kube-post-application-update', + 'success_state': 'post-updated-apps', + 'fail_state': 'post-updating-apps-failed'}, + ], + } + def _kube_upgrade_complete_stage(self): return { 'name': 'kube-upgrade-complete', @@ -496,6 +517,7 @@ class ApplyStageMixin(object): storage_list=None, add_start=True, add_download=True, + add_pre_app_update=True, add_networking=True, add_storage=True, add_cordon=True, @@ -503,6 +525,7 @@ class ApplyStageMixin(object): add_second_control_plane=True, add_kubelets=True, add_uncordon=True, + add_post_app_update=True, add_complete=True, add_cleanup=True): """The order of the host_list determines the kubelets""" @@ -518,6 +541,8 @@ class ApplyStageMixin(object): stages.append(self._kube_upgrade_start_stage()) if add_download: stages.append(self._kube_upgrade_download_images_stage()) + if add_pre_app_update: + stages.append(self._kube_pre_application_update_stage()) if add_networking: stages.append(self._kube_upgrade_networking_stage()) if add_storage: @@ -537,6 +562,8 @@ class ApplyStageMixin(object): worker_list)) if add_uncordon: stages.append(self._kube_host_uncordon_stage()) + if add_post_app_update: + stages.append(self._kube_post_application_update_stage()) if add_complete: stages.append(self._kube_upgrade_complete_stage()) if add_cleanup: @@ -624,39 +651,6 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, self.worker_list = [] self.storage_list = [] - def test_resume_after_starting_failed(self): - """ - Test the kube_upgrade strategy creation when the upgrade had previously - stopped with 'upgrade-starting-failed' - It is expected to resume at the 'upgrade-starting' stage - """ - kube_upgrade = self._create_kube_upgrade_obj( - KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTING_FAILED, - self.default_from_version, - self.default_to_version) - stages = [ - self._kube_upgrade_start_stage(), - self._kube_upgrade_download_images_stage(), - self._kube_upgrade_networking_stage(), - self._kube_upgrade_storage_stage(), - ] - if self.is_simplex(): - stages.append(self._kube_host_cordon_stage()) - for ver in self.kube_versions: - stages.append(self._kube_upgrade_first_control_plane_stage(ver)) - stages.extend(self._kube_upgrade_kubelet_stages( - ver, - self.std_controller_list, - self.aio_controller_list, - self.worker_list)) - if self.is_simplex(): - stages.append(self._kube_host_uncordon_stage()) - stages.extend([ - self._kube_upgrade_complete_stage(), - self._kube_upgrade_cleanup_stage(), - ]) - self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) - def test_resume_after_download_images_failed(self): """ Test the kube_upgrade strategy creation when the upgrade had previously @@ -669,6 +663,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, self.default_to_version) stages = [ self._kube_upgrade_download_images_stage(), + self._kube_pre_application_update_stage(), self._kube_upgrade_networking_stage(), self._kube_upgrade_storage_stage(), ] @@ -684,6 +679,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, if self.is_simplex(): stages.append(self._kube_host_uncordon_stage()) stages.extend([ + self._kube_post_application_update_stage(), self._kube_upgrade_complete_stage(), self._kube_upgrade_cleanup_stage(), ]) @@ -693,12 +689,79 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, """ Test the kube_upgrade strategy creation when the upgrade had previously stopped with 'downloaded-images' - It is expected to resume at the 'first control plane' stage. + It is expected to resume at the 'pre application update' stage. """ kube_upgrade = self._create_kube_upgrade_obj( KUBE_UPGRADE_STATE.KUBE_UPGRADE_DOWNLOADED_IMAGES, self.default_from_version, self.default_to_version) + stages = [ + self._kube_pre_application_update_stage(), + self._kube_upgrade_networking_stage(), + self._kube_upgrade_storage_stage(), + ] + if self.is_simplex(): + stages.append(self._kube_host_cordon_stage()) + for ver in self.kube_versions: + stages.append(self._kube_upgrade_first_control_plane_stage( + ver)) + stages.extend(self._kube_upgrade_kubelet_stages( + ver, + self.std_controller_list, + self.aio_controller_list, + self.worker_list)) + if self.is_simplex(): + stages.append(self._kube_host_uncordon_stage()) + stages.extend([ + self._kube_post_application_update_stage(), + self._kube_upgrade_complete_stage(), + self._kube_upgrade_cleanup_stage(), + ]) + self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) + + def test_resume_after_pre_app_update_failed(self): + """ + Test the kube_upgrade strategy creation when the upgrade had previously + stopped with 'pre-updating-apps-failed' + It is expected to resume at the 'pre-updating-apps' stage + """ + kube_upgrade = self._create_kube_upgrade_obj( + KUBE_UPGRADE_STATE.KUBE_PRE_UPDATING_APPS_FAILED, + self.default_from_version, + self.default_to_version) + stages = [ + self._kube_pre_application_update_stage(), + self._kube_upgrade_networking_stage(), + self._kube_upgrade_storage_stage(), + ] + if self.is_simplex(): + stages.append(self._kube_host_cordon_stage()) + for ver in self.kube_versions: + stages.append(self._kube_upgrade_first_control_plane_stage(ver)) + stages.extend(self._kube_upgrade_kubelet_stages( + ver, + self.std_controller_list, + self.aio_controller_list, + self.worker_list)) + if self.is_simplex(): + stages.append(self._kube_host_uncordon_stage()) + stages.extend([ + self._kube_post_application_update_stage(), + self._kube_upgrade_complete_stage(), + self._kube_upgrade_cleanup_stage(), + ]) + self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages) + + def test_resume_after_pre_app_update_succeeded(self): + """ + Test the kube_upgrade strategy creation when the upgrade had previously + stopped with 'pre-updated-apps' + It is expected to resume at the 'upgrade networking' stage. + """ + kube_upgrade = self._create_kube_upgrade_obj( + KUBE_UPGRADE_STATE.KUBE_PRE_UPDATED_APPS, + self.default_from_version, + self.default_to_version) stages = [ self._kube_upgrade_networking_stage(), self._kube_upgrade_storage_stage(), @@ -716,6 +779,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, if self.is_simplex(): stages.append(self._kube_host_uncordon_stage()) stages.extend([ + self._kube_post_application_update_stage(), self._kube_upgrade_complete_stage(), self._kube_upgrade_cleanup_stage(), ]) @@ -743,6 +807,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, if self.is_simplex(): stages.append(self._kube_host_uncordon_stage()) stages.extend([ + self._kube_post_application_update_stage(), self._kube_upgrade_complete_stage(), self._kube_upgrade_cleanup_stage(), ]) @@ -768,6 +833,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, if self.is_simplex(): stages.append(self._kube_host_uncordon_stage()) stages.extend([ + self._kube_post_application_update_stage(), self._kube_upgrade_complete_stage(), self._kube_upgrade_cleanup_stage(), ]) @@ -800,6 +866,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, if self.is_simplex(): stages.append(self._kube_host_uncordon_stage()) stages.extend([ + self._kube_post_application_update_stage(), self._kube_upgrade_complete_stage(), self._kube_upgrade_cleanup_stage(), ]) @@ -831,6 +898,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, if self.is_simplex(): stages.append(self._kube_host_uncordon_stage()) stages.extend([ + self._kube_post_application_update_stage(), self._kube_upgrade_complete_stage(), self._kube_upgrade_cleanup_stage(), ]) @@ -862,6 +930,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, if self.is_simplex(): stages.append(self._kube_host_uncordon_stage()) stages.extend([ + self._kube_post_application_update_stage(), self._kube_upgrade_complete_stage(), self._kube_upgrade_cleanup_stage(), ]) @@ -891,6 +960,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, if self.is_simplex(): stages.append(self._kube_host_uncordon_stage()) stages.extend([ + self._kube_post_application_update_stage(), self._kube_upgrade_complete_stage(), self._kube_upgrade_cleanup_stage(), ]) @@ -918,6 +988,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, if self.is_simplex(): stages.append(self._kube_host_uncordon_stage()) stages.extend([ + self._kube_post_application_update_stage(), self._kube_upgrade_complete_stage(), self._kube_upgrade_cleanup_stage(), ]) @@ -945,6 +1016,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase, if self.is_simplex(): stages.append(self._kube_host_uncordon_stage()) stages.extend([ + self._kube_post_application_update_stage(), self._kube_upgrade_complete_stage(), self._kube_upgrade_cleanup_stage(), ]) diff --git a/nfv/nfv-vim/nfv_vim/nfvi/__init__.py b/nfv/nfv-vim/nfv_vim/nfvi/__init__.py index 859edea7..34d1d12a 100755 --- a/nfv/nfv-vim/nfv_vim/nfvi/__init__.py +++ b/nfv/nfv-vim/nfv_vim/nfvi/__init__.py @@ -117,6 +117,8 @@ from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_host_cordon # no from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_host_uncordon # noqa: F401 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_host_upgrade_control_plane # noqa: F401 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_host_upgrade_kubelet # noqa: F401 +from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_post_application_update # noqa: F401 +from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_pre_application_update # noqa: F401 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_rootca_update_abort # noqa: F401 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_rootca_update_complete # noqa: F401 from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_rootca_update_generate_cert # noqa: F401 diff --git a/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_infrastructure_module.py b/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_infrastructure_module.py index 466b7b1c..d25cae70 100755 --- a/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_infrastructure_module.py +++ b/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_infrastructure_module.py @@ -321,6 +321,26 @@ def nfvi_kube_upgrade_start(to_version, force, alarm_ignore_list, callback): return cmd_id +def nfvi_kube_pre_application_update(callback): + """ + Kube Upgrade Pre Application Update + """ + cmd_id = _infrastructure_plugin.invoke_plugin( + 'kube_pre_application_update', + callback=callback) + return cmd_id + + +def nfvi_kube_post_application_update(callback): + """ + Kube Upgrade Post Application Update + """ + cmd_id = _infrastructure_plugin.invoke_plugin( + 'kube_post_application_update', + callback=callback) + return cmd_id + + def nfvi_get_kube_host_upgrade_list(callback): """ Get kube host upgrade list diff --git a/nfv/nfv-vim/nfv_vim/nfvi/objects/v1/_kube_upgrade.py b/nfv/nfv-vim/nfv_vim/nfvi/objects/v1/_kube_upgrade.py index 04d50364..fe956e2b 100755 --- a/nfv/nfv-vim/nfv_vim/nfvi/objects/v1/_kube_upgrade.py +++ b/nfv/nfv-vim/nfv_vim/nfvi/objects/v1/_kube_upgrade.py @@ -37,12 +37,16 @@ class KubeUpgradeState(Constants): Maintaining the same order as defined in kubernetes.py """ - KUBE_UPGRADE_STARTING = Constant('upgrade-starting') - KUBE_UPGRADE_STARTING_FAILED = Constant('upgrade-starting-failed') KUBE_UPGRADE_STARTED = Constant('upgrade-started') KUBE_UPGRADE_DOWNLOADING_IMAGES = Constant('downloading-images') KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED = Constant('downloading-images-failed') KUBE_UPGRADE_DOWNLOADED_IMAGES = Constant('downloaded-images') + KUBE_PRE_UPDATING_APPS = Constant('pre-updating-apps') + KUBE_PRE_UPDATING_APPS_FAILED = Constant('pre-updating-apps-failed') + KUBE_PRE_UPDATED_APPS = Constant('pre-updated-apps') + KUBE_POST_UPDATING_APPS = Constant('post-updating-apps') + KUBE_POST_UPDATING_APPS_FAILED = Constant('post-updating-apps-failed') + KUBE_POST_UPDATED_APPS = Constant('post-updated-apps') KUBE_UPGRADING_NETWORKING = Constant('upgrading-networking') KUBE_UPGRADING_NETWORKING_FAILED = Constant('upgrading-networking-failed') KUBE_UPGRADED_NETWORKING = Constant('upgraded-networking') diff --git a/nfv/nfv-vim/nfv_vim/strategy/__init__.py b/nfv/nfv-vim/nfv_vim/strategy/__init__.py index 50fa177f..03cb2855 100755 --- a/nfv/nfv-vim/nfv_vim/strategy/__init__.py +++ b/nfv/nfv-vim/nfv_vim/strategy/__init__.py @@ -21,6 +21,8 @@ from nfv_vim.strategy._strategy_steps import KubeHostCordonStep # noqa: F401 from nfv_vim.strategy._strategy_steps import KubeHostUncordonStep # noqa: F401 from nfv_vim.strategy._strategy_steps import KubeHostUpgradeControlPlaneStep # noqa: F401 from nfv_vim.strategy._strategy_steps import KubeHostUpgradeKubeletStep # noqa: F401 +from nfv_vim.strategy._strategy_steps import KubePostApplicationUpdateStep # noqa: F401 +from nfv_vim.strategy._strategy_steps import KubePreApplicationUpdateStep # noqa: F401 from nfv_vim.strategy._strategy_steps import KubeRootcaUpdateCompleteStep # noqa: F401 from nfv_vim.strategy._strategy_steps import KubeRootcaUpdateGenerateCertStep # noqa: F401 from nfv_vim.strategy._strategy_steps import KubeRootcaUpdateHostTrustBothcasStep # noqa: F401 diff --git a/nfv/nfv-vim/nfv_vim/strategy/_strategy.py b/nfv/nfv-vim/nfv_vim/strategy/_strategy.py index 7db764f6..04d6e7cb 100755 --- a/nfv/nfv-vim/nfv_vim/strategy/_strategy.py +++ b/nfv/nfv-vim/nfv_vim/strategy/_strategy.py @@ -3454,7 +3454,23 @@ class KubeUpgradeStrategy(SwUpdateStrategy, strategy.STRATEGY_STAGE_NAME.KUBE_UPGRADE_DOWNLOAD_IMAGES) stage.add_step(strategy.KubeUpgradeDownloadImagesStep()) self.apply_phase.add_stage(stage) - # Next stage after download images is upgrade networking + # Next stage after download images is pre application update + self._add_kube_pre_application_update_stage() + + def _add_kube_pre_application_update_stage(self): + """ + Add kube pre application update stage. + This stage only occurs after download images + It then proceeds to the next stage + """ + from nfv_vim import strategy + + stage = strategy.StrategyStage( + strategy.STRATEGY_STAGE_NAME.KUBE_PRE_APPLICATION_UPDATE) + stage.add_step(strategy.KubePreApplicationUpdateStep()) + self.apply_phase.add_stage(stage) + + # Next stage after pre application update is upgrade networking self._add_kube_upgrade_networking_stage() def _add_kube_upgrade_networking_stage(self): @@ -3599,8 +3615,8 @@ class KubeUpgradeStrategy(SwUpdateStrategy, nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_HOST_UNCORDON_FAILED) ) self.apply_phase.add_stage(stage) - # after this loop is kube upgrade complete stage - self._add_kube_upgrade_complete_stage() + # after this loop is kube post application update stage + self._add_kube_post_application_update_stage() def _add_kube_upgrade_first_control_plane_stage(self, first_host, kube_ver): """Add first controller control plane kube upgrade stage""" @@ -3729,10 +3745,26 @@ class KubeUpgradeStrategy(SwUpdateStrategy, self.report_build_failure(reason) return + def _add_kube_post_application_update_stage(self): + """ + Add kube post application update stage. + This stage occurs after all kubelets are upgraded and hosts uncordoned + It then proceeds to the next stage + """ + from nfv_vim import strategy + + stage = strategy.StrategyStage( + strategy.STRATEGY_STAGE_NAME.KUBE_POST_APPLICATION_UPDATE) + stage.add_step(strategy.KubePostApplicationUpdateStep()) + self.apply_phase.add_stage(stage) + + # Next stage after post application update is upgrade complete + self._add_kube_upgrade_complete_stage() + def _add_kube_upgrade_complete_stage(self): """ Add kube upgrade complete strategy stage - This stage occurs after all kubelets are upgraded + This stage occurs after apps are post updated """ from nfv_vim import strategy stage = strategy.StrategyStage( @@ -3806,10 +3838,6 @@ class KubeUpgradeStrategy(SwUpdateStrategy, # Note: there are no resume states for actions that are still running # ie: KUBE_UPGRADE_DOWNLOADING_IMAGES RESUME_STATE = { - # If upgrade start failed, allow to restart - nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTING_FAILED: - self._add_kube_upgrade_start_stage, - # after upgrade-started -> download images nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTED: self._add_kube_upgrade_download_images_stage, @@ -3818,8 +3846,16 @@ class KubeUpgradeStrategy(SwUpdateStrategy, nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED: self._add_kube_upgrade_download_images_stage, - # After downloading images -> upgrade networking + # After downloading images -> pre update applications nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_DOWNLOADED_IMAGES: + self._add_kube_pre_application_update_stage, + + # if pre updating apps failed, resume at pre updating apps + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_PRE_UPDATING_APPS_FAILED: + self._add_kube_pre_application_update_stage, + + # After pre updating apps -> upgrade networking + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_PRE_UPDATED_APPS: self._add_kube_upgrade_networking_stage, # if networking state failed, resync at networking state @@ -3870,8 +3906,16 @@ class KubeUpgradeStrategy(SwUpdateStrategy, nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_HOST_UNCORDON_FAILED: self._add_kube_host_uncordon_stage, - # If the state is uncordon-complete, resume at complete stage + # After uncordon-complete -> post update apps nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_HOST_UNCORDON_COMPLETE: + self._add_kube_post_application_update_stage, + + # If post updating apps failed, resume at post application update stage + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_POST_UPDATING_APPS_FAILED: + self._add_kube_post_application_update_stage, + + # After post updating apps -> upgrade complete + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_POST_UPDATED_APPS: self._add_kube_upgrade_complete_stage, # upgrade is completed, delete the upgrade diff --git a/nfv/nfv-vim/nfv_vim/strategy/_strategy_stages.py b/nfv/nfv-vim/nfv_vim/strategy/_strategy_stages.py index dd02a0d9..07c9c0f2 100755 --- a/nfv/nfv-vim/nfv_vim/strategy/_strategy_stages.py +++ b/nfv/nfv-vim/nfv_vim/strategy/_strategy_stages.py @@ -63,6 +63,7 @@ class StrategyStageNames(Constants): KUBE_UPGRADE_QUERY = Constant('kube-upgrade-query') KUBE_UPGRADE_START = Constant('kube-upgrade-start') KUBE_UPGRADE_DOWNLOAD_IMAGES = Constant('kube-upgrade-download-images') + KUBE_PRE_APPLICATION_UPDATE = Constant('kube-pre-application-update') KUBE_UPGRADE_FIRST_CONTROL_PLANE = \ Constant('kube-upgrade-first-control-plane') KUBE_UPGRADE_NETWORKING = Constant('kube-upgrade-networking') @@ -73,6 +74,7 @@ class StrategyStageNames(Constants): KUBE_UPGRADE_KUBELETS_CONTROLLERS = \ Constant('kube-upgrade-kubelets-controllers') KUBE_UPGRADE_KUBELETS_WORKERS = Constant('kube-upgrade-kubelets-workers') + KUBE_POST_APPLICATION_UPDATE = Constant('kube-post-application-update') KUBE_UPGRADE_COMPLETE = Constant('kube-upgrade-complete') KUBE_UPGRADE_CLEANUP = Constant('kube-upgrade-cleanup') diff --git a/nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py b/nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py index 245c4052..bc7a2d8e 100755 --- a/nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py +++ b/nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py @@ -87,6 +87,8 @@ class StrategyStepNames(Constants): KUBE_HOST_UPGRADE_CONTROL_PLANE = \ Constant('kube-host-upgrade-control-plane') KUBE_HOST_UPGRADE_KUBELET = Constant('kube-host-upgrade-kubelet') + KUBE_PRE_APPLICATION_UPDATE = Constant('kube-pre-application-update') + KUBE_POST_APPLICATION_UPDATE = Constant('kube-post-application-update') # system config update specific steps QUERY_SYSTEM_CONFIG_UPDATE_HOSTS = Constant('query-system-config-update-hosts') SYSTEM_CONFIG_UPDATE_HOSTS = Constant('system-config-update-hosts') @@ -4172,8 +4174,7 @@ class KubeUpgradeStartStep(AbstractKubeUpgradeStep): super(KubeUpgradeStartStep, self).__init__( STRATEGY_STEP_NAME.KUBE_UPGRADE_START, nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTED, - nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTING_FAILED, - timeout_in_secs=1800) + None) # there is no failure state if upgrade-start fails # next 2 attributes must be persisted through from_dict/as_dict self._to_version = to_version self._force = force @@ -4209,9 +4210,13 @@ class KubeUpgradeStartStep(AbstractKubeUpgradeStep): response = (yield) DLOG.debug("%s callback response=%s." % (self._name, response)) + # kube-upgrade-start will return a result when it completes, + # so we do not want to use handle_event if response['completed']: if self.strategy is not None: self.strategy.nfvi_kube_upgrade = response['result-data'] + result = strategy.STRATEGY_STEP_RESULT.SUCCESS + self.stage.step_complete(result, "") else: result = strategy.STRATEGY_STEP_RESULT.FAILED self.stage.step_complete(result, response['reason']) @@ -4401,6 +4406,88 @@ class KubeUpgradeDownloadImagesStep(AbstractKubeUpgradeStep): return strategy.STRATEGY_STEP_RESULT.WAIT, "" +class KubePreApplicationUpdateStep(AbstractKubeUpgradeStep): + """Kube Pre Application Update - Strategy Step""" + + def __init__(self): + from nfv_vim import nfvi + super(KubePreApplicationUpdateStep, self).__init__( + STRATEGY_STEP_NAME.KUBE_PRE_APPLICATION_UPDATE, + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_PRE_UPDATED_APPS, + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_PRE_UPDATING_APPS_FAILED, + timeout_in_secs=1800) + + def abort(self): + """ + Returns the abort step related to this step + """ + return [KubeUpgradeAbortStep()] + + @coroutine + def _response_callback(self): + """Kube Pre Application Update - Callback""" + + response = (yield) + DLOG.debug("%s callback response=%s." % (self._name, response)) + + if response['completed']: + if self.strategy is not None: + self.strategy.nfvi_kube_upgrade = response['result-data'] + else: + result = strategy.STRATEGY_STEP_RESULT.FAILED + self.stage.step_complete(result, response['reason']) + + def apply(self): + """Kube Pre Application Update """ + + from nfv_vim import nfvi + DLOG.info("Step (%s) apply." % self._name) + + nfvi.nfvi_kube_pre_application_update(self._response_callback()) + return strategy.STRATEGY_STEP_RESULT.WAIT, "" + + +class KubePostApplicationUpdateStep(AbstractKubeUpgradeStep): + """Kube Post Application Update - Strategy Step""" + + def __init__(self): + from nfv_vim import nfvi + super(KubePostApplicationUpdateStep, self).__init__( + STRATEGY_STEP_NAME.KUBE_POST_APPLICATION_UPDATE, + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_POST_UPDATED_APPS, + nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_POST_UPDATING_APPS_FAILED, + timeout_in_secs=1800) + + def abort(self): + """ + Returns the abort step related to this step + """ + return [KubeUpgradeAbortStep()] + + @coroutine + def _response_callback(self): + """Kube Post Application Update - Callback""" + + response = (yield) + DLOG.debug("%s callback response=%s." % (self._name, response)) + + if response['completed']: + if self.strategy is not None: + self.strategy.nfvi_kube_upgrade = response['result-data'] + else: + result = strategy.STRATEGY_STEP_RESULT.FAILED + self.stage.step_complete(result, response['reason']) + + def apply(self): + """Kube Post Application Update """ + + from nfv_vim import nfvi + DLOG.info("Step (%s) apply." % self._name) + + nfvi.nfvi_kube_post_application_update(self._response_callback()) + return strategy.STRATEGY_STEP_RESULT.WAIT, "" + + class KubeUpgradeNetworkingStep(AbstractKubeUpgradeStep): """Kube Upgrade Networking - Strategy Step""" @@ -4932,6 +5019,8 @@ def strategy_step_rebuild_from_dict(data): STRATEGY_STEP_NAME.KUBE_UPGRADE_NETWORKING: KubeUpgradeNetworkingStep, STRATEGY_STEP_NAME.KUBE_UPGRADE_STORAGE: KubeUpgradeStorageStep, STRATEGY_STEP_NAME.KUBE_UPGRADE_START: KubeUpgradeStartStep, + STRATEGY_STEP_NAME.KUBE_PRE_APPLICATION_UPDATE: KubePreApplicationUpdateStep, + STRATEGY_STEP_NAME.KUBE_POST_APPLICATION_UPDATE: KubePostApplicationUpdateStep, STRATEGY_STEP_NAME.QUERY_KUBE_HOST_UPGRADE: QueryKubeHostUpgradeStep, STRATEGY_STEP_NAME.QUERY_KUBE_UPGRADE: QueryKubeUpgradeStep, STRATEGY_STEP_NAME.QUERY_KUBE_VERSIONS: QueryKubeVersionsStep,