Add new application update steps for k8s upgrades
Add two new steps to the orchestrated Kubernetes upgrade: * 'pre-application-update': update StarlingX applications before the new Kubernetes version is deployed. * 'post-application-update': update Starlingx applications after the new Kuberneter version is deployed. This was implemented according to the changes introduced to sysinv by the following commit: https://review.opendev.org/c/starlingx/config/+/909172 Unit tests have also been updated to account for the new steps. Test Plan: PASS: build-pkgs -a && build-image AIO-SX Test Cases: PASS: Run an orchestrated Kubernetes upgrade with no apps to be updated. PASS: Run a single-version orchestrated Kubernetes upgrade with apps to be updated during the pre-application-update and post-application-update steps. PASS: Run a multi-version orchestrated Kubernetes upgrade with apps to be updated during the pre-application-update and post-application-update steps. PASS: Run an orchestrated Kubernetes upgrade that fails during the pre-application-update step due to a missing metadata.yaml file. Confirm that the upgrade was successfully aborted. PASS: Run an orchestrated Kubernetes upgrade that fails during the post-application-update step due to a missing metadata.yaml file. Confirm that the upgrade was successfully aborted. AIO-DX Test Cases: PASS: Run an orchestrated Kubernetes upgrade with no apps to be updated. PASS: Run a single-version orchestrated Kubernetes upgrade with apps to be updated during the pre-application-update and post-application-update steps. PASS: Run an orchestrated Kubernetes upgrade that fails during the pre-application-update step due to a missing metadata.yaml file. Fix the application tarball and resume the upgrade. Confirm that the upgrade resumes from the same step. Confirm that the upgrade finishes successfully. PASS: Run an orchestrated Kubernetes upgrade that fails during the post-application-update step due to a missing metadata.yaml file. Fix the application tarball and resume the upgrade. Confirm that the upgrade resumes from the same step. Confirm that the upgrade finishes successfully. Story: 2010929 Task: 49791 Depends-On: https://review.opendev.org/c/starlingx/config/+/909172 Change-Id: I5217db1cb744320979afcd161cb436bf089ccc9d Signed-off-by: Igor Soares <Igor.PiresSoares@windriver.com>
This commit is contained in:
parent
6142d9f116
commit
3e06391c30
@ -2017,6 +2017,124 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
|
||||
callback.send(response)
|
||||
callback.close()
|
||||
|
||||
def kube_pre_application_update(self, future, callback):
|
||||
"""
|
||||
Start kube pre application update
|
||||
"""
|
||||
response = dict()
|
||||
response['completed'] = False
|
||||
response['reason'] = ''
|
||||
action_type = 'kube-pre-application-update'
|
||||
|
||||
try:
|
||||
future.set_timeouts(config.CONF.get('nfvi-timeouts', None))
|
||||
|
||||
if self._platform_token is None or \
|
||||
self._platform_token.is_expired():
|
||||
future.work(openstack.get_token, self._platform_directory)
|
||||
future.result = (yield)
|
||||
|
||||
if not future.result.is_complete() or \
|
||||
future.result.data is None:
|
||||
DLOG.error("OpenStack get-token did not complete.")
|
||||
return
|
||||
|
||||
self._platform_token = future.result.data
|
||||
|
||||
future.work(sysinv.kube_pre_application_update,
|
||||
self._platform_token)
|
||||
future.result = (yield)
|
||||
|
||||
if not future.result.is_complete():
|
||||
DLOG.error("%s did not complete." % action_type)
|
||||
return
|
||||
|
||||
kube_upgrade_data = future.result.data
|
||||
kube_upgrade_obj = nfvi.objects.v1.KubeUpgrade(
|
||||
kube_upgrade_data['state'],
|
||||
kube_upgrade_data['from_version'],
|
||||
kube_upgrade_data['to_version'])
|
||||
|
||||
response['result-data'] = kube_upgrade_obj
|
||||
response['completed'] = True
|
||||
|
||||
except exceptions.OpenStackRestAPIException as e:
|
||||
if httplib.UNAUTHORIZED == e.http_status_code:
|
||||
response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED
|
||||
if self._platform_token is not None:
|
||||
self._platform_token.set_expired()
|
||||
|
||||
else:
|
||||
DLOG.exception("Caught API exception while trying %s. error=%s"
|
||||
% (action_type, e))
|
||||
|
||||
except Exception as e:
|
||||
DLOG.exception("Caught exception while trying %s. error=%s"
|
||||
% (action_type, e))
|
||||
|
||||
finally:
|
||||
callback.send(response)
|
||||
callback.close()
|
||||
|
||||
def kube_post_application_update(self, future, callback):
|
||||
"""
|
||||
Start kube post application update
|
||||
"""
|
||||
response = dict()
|
||||
response['completed'] = False
|
||||
response['reason'] = ''
|
||||
action_type = 'kube-post-application-update'
|
||||
|
||||
try:
|
||||
future.set_timeouts(config.CONF.get('nfvi-timeouts', None))
|
||||
|
||||
if self._platform_token is None or \
|
||||
self._platform_token.is_expired():
|
||||
future.work(openstack.get_token, self._platform_directory)
|
||||
future.result = (yield)
|
||||
|
||||
if not future.result.is_complete() or \
|
||||
future.result.data is None:
|
||||
DLOG.error("OpenStack get-token did not complete.")
|
||||
return
|
||||
|
||||
self._platform_token = future.result.data
|
||||
|
||||
future.work(sysinv.kube_post_application_update,
|
||||
self._platform_token)
|
||||
future.result = (yield)
|
||||
|
||||
if not future.result.is_complete():
|
||||
DLOG.error("%s did not complete." % action_type)
|
||||
return
|
||||
|
||||
kube_upgrade_data = future.result.data
|
||||
kube_upgrade_obj = nfvi.objects.v1.KubeUpgrade(
|
||||
kube_upgrade_data['state'],
|
||||
kube_upgrade_data['from_version'],
|
||||
kube_upgrade_data['to_version'])
|
||||
|
||||
response['result-data'] = kube_upgrade_obj
|
||||
response['completed'] = True
|
||||
|
||||
except exceptions.OpenStackRestAPIException as e:
|
||||
if httplib.UNAUTHORIZED == e.http_status_code:
|
||||
response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED
|
||||
if self._platform_token is not None:
|
||||
self._platform_token.set_expired()
|
||||
|
||||
else:
|
||||
DLOG.exception("Caught API exception while trying %s. error=%s"
|
||||
% (action_type, e))
|
||||
|
||||
except Exception as e:
|
||||
DLOG.exception("Caught exception while trying %s. error=%s"
|
||||
% (action_type, e))
|
||||
|
||||
finally:
|
||||
callback.send(response)
|
||||
callback.close()
|
||||
|
||||
def kube_upgrade_networking(self, future, callback):
|
||||
"""
|
||||
Start kube upgrade networking
|
||||
|
@ -436,6 +436,13 @@ def kube_upgrade_download_images(token):
|
||||
return _patch_kube_upgrade_state(token, "downloading-images")
|
||||
|
||||
|
||||
def kube_pre_application_update(token):
|
||||
"""
|
||||
Ask System Inventory to kube pre application update
|
||||
"""
|
||||
return _patch_kube_upgrade_state(token, "pre-updating-apps")
|
||||
|
||||
|
||||
def kube_upgrade_networking(token):
|
||||
"""
|
||||
Ask System Inventory to kube upgrade networking
|
||||
@ -583,6 +590,13 @@ def upgrade_activate(token):
|
||||
return response
|
||||
|
||||
|
||||
def kube_post_application_update(token):
|
||||
"""
|
||||
Ask System Inventory to kube post application update
|
||||
"""
|
||||
return _patch_kube_upgrade_state(token, "post-updating-apps")
|
||||
|
||||
|
||||
def upgrade_complete(token):
|
||||
"""
|
||||
Ask System Inventory to complete an upgrade
|
||||
|
@ -269,8 +269,7 @@ class ApplyStageMixin(object):
|
||||
'total_steps': 1,
|
||||
'steps': [
|
||||
{'name': 'kube-upgrade-start',
|
||||
'success_state': 'upgrade-started',
|
||||
'fail_state': 'upgrade-starting-failed'},
|
||||
'success_state': 'upgrade-started'},
|
||||
],
|
||||
}
|
||||
|
||||
@ -285,6 +284,17 @@ class ApplyStageMixin(object):
|
||||
],
|
||||
}
|
||||
|
||||
def _kube_pre_application_update_stage(self):
|
||||
return {
|
||||
'name': 'kube-pre-application-update',
|
||||
'total_steps': 1,
|
||||
'steps': [
|
||||
{'name': 'kube-pre-application-update',
|
||||
'success_state': 'pre-updated-apps',
|
||||
'fail_state': 'pre-updating-apps-failed'},
|
||||
],
|
||||
}
|
||||
|
||||
def _kube_host_cordon_stage(self, ver="N/A"):
|
||||
return {
|
||||
'name': 'kube-host-cordon',
|
||||
@ -362,6 +372,17 @@ class ApplyStageMixin(object):
|
||||
],
|
||||
}
|
||||
|
||||
def _kube_post_application_update_stage(self):
|
||||
return {
|
||||
'name': 'kube-post-application-update',
|
||||
'total_steps': 1,
|
||||
'steps': [
|
||||
{'name': 'kube-post-application-update',
|
||||
'success_state': 'post-updated-apps',
|
||||
'fail_state': 'post-updating-apps-failed'},
|
||||
],
|
||||
}
|
||||
|
||||
def _kube_upgrade_cleanup_stage(self):
|
||||
return {
|
||||
'name': 'kube-upgrade-cleanup',
|
||||
@ -496,6 +517,7 @@ class ApplyStageMixin(object):
|
||||
storage_list=None,
|
||||
add_start=True,
|
||||
add_download=True,
|
||||
add_pre_app_update=True,
|
||||
add_networking=True,
|
||||
add_storage=True,
|
||||
add_cordon=True,
|
||||
@ -503,6 +525,7 @@ class ApplyStageMixin(object):
|
||||
add_second_control_plane=True,
|
||||
add_kubelets=True,
|
||||
add_uncordon=True,
|
||||
add_post_app_update=True,
|
||||
add_complete=True,
|
||||
add_cleanup=True):
|
||||
"""The order of the host_list determines the kubelets"""
|
||||
@ -518,6 +541,8 @@ class ApplyStageMixin(object):
|
||||
stages.append(self._kube_upgrade_start_stage())
|
||||
if add_download:
|
||||
stages.append(self._kube_upgrade_download_images_stage())
|
||||
if add_pre_app_update:
|
||||
stages.append(self._kube_pre_application_update_stage())
|
||||
if add_networking:
|
||||
stages.append(self._kube_upgrade_networking_stage())
|
||||
if add_storage:
|
||||
@ -539,6 +564,8 @@ class ApplyStageMixin(object):
|
||||
stages.append(self._kube_host_uncordon_stage())
|
||||
if add_complete:
|
||||
stages.append(self._kube_upgrade_complete_stage())
|
||||
if add_post_app_update:
|
||||
stages.append(self._kube_post_application_update_stage())
|
||||
if add_cleanup:
|
||||
stages.append(self._kube_upgrade_cleanup_stage())
|
||||
return stages
|
||||
@ -582,7 +609,7 @@ class ApplyStageMixin(object):
|
||||
"""
|
||||
Test the kube_upgrade strategy creation when the upgrade had previously
|
||||
stopped after upgrade-completed.
|
||||
It is expected to resume at the cleanup stage
|
||||
It is expected to resume at the post application update stage
|
||||
"""
|
||||
kube_upgrade = self._create_kube_upgrade_obj(
|
||||
KUBE_UPGRADE_STATE.KUBE_UPGRADE_COMPLETE,
|
||||
@ -590,6 +617,7 @@ class ApplyStageMixin(object):
|
||||
self.default_to_version)
|
||||
# not using build_stage_list utility since the list of stages is small
|
||||
stages = [
|
||||
self._kube_post_application_update_stage(),
|
||||
self._kube_upgrade_cleanup_stage(),
|
||||
]
|
||||
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
|
||||
@ -624,39 +652,6 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
|
||||
self.worker_list = []
|
||||
self.storage_list = []
|
||||
|
||||
def test_resume_after_starting_failed(self):
|
||||
"""
|
||||
Test the kube_upgrade strategy creation when the upgrade had previously
|
||||
stopped with 'upgrade-starting-failed'
|
||||
It is expected to resume at the 'upgrade-starting' stage
|
||||
"""
|
||||
kube_upgrade = self._create_kube_upgrade_obj(
|
||||
KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTING_FAILED,
|
||||
self.default_from_version,
|
||||
self.default_to_version)
|
||||
stages = [
|
||||
self._kube_upgrade_start_stage(),
|
||||
self._kube_upgrade_download_images_stage(),
|
||||
self._kube_upgrade_networking_stage(),
|
||||
self._kube_upgrade_storage_stage(),
|
||||
]
|
||||
if self.is_simplex():
|
||||
stages.append(self._kube_host_cordon_stage())
|
||||
for ver in self.kube_versions:
|
||||
stages.append(self._kube_upgrade_first_control_plane_stage(ver))
|
||||
stages.extend(self._kube_upgrade_kubelet_stages(
|
||||
ver,
|
||||
self.std_controller_list,
|
||||
self.aio_controller_list,
|
||||
self.worker_list))
|
||||
if self.is_simplex():
|
||||
stages.append(self._kube_host_uncordon_stage())
|
||||
stages.extend([
|
||||
self._kube_upgrade_complete_stage(),
|
||||
self._kube_upgrade_cleanup_stage(),
|
||||
])
|
||||
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
|
||||
|
||||
def test_resume_after_download_images_failed(self):
|
||||
"""
|
||||
Test the kube_upgrade strategy creation when the upgrade had previously
|
||||
@ -669,6 +664,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
|
||||
self.default_to_version)
|
||||
stages = [
|
||||
self._kube_upgrade_download_images_stage(),
|
||||
self._kube_pre_application_update_stage(),
|
||||
self._kube_upgrade_networking_stage(),
|
||||
self._kube_upgrade_storage_stage(),
|
||||
]
|
||||
@ -685,6 +681,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
|
||||
stages.append(self._kube_host_uncordon_stage())
|
||||
stages.extend([
|
||||
self._kube_upgrade_complete_stage(),
|
||||
self._kube_post_application_update_stage(),
|
||||
self._kube_upgrade_cleanup_stage(),
|
||||
])
|
||||
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
|
||||
@ -693,12 +690,79 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
|
||||
"""
|
||||
Test the kube_upgrade strategy creation when the upgrade had previously
|
||||
stopped with 'downloaded-images'
|
||||
It is expected to resume at the 'first control plane' stage.
|
||||
It is expected to resume at the 'pre application update' stage.
|
||||
"""
|
||||
kube_upgrade = self._create_kube_upgrade_obj(
|
||||
KUBE_UPGRADE_STATE.KUBE_UPGRADE_DOWNLOADED_IMAGES,
|
||||
self.default_from_version,
|
||||
self.default_to_version)
|
||||
stages = [
|
||||
self._kube_pre_application_update_stage(),
|
||||
self._kube_upgrade_networking_stage(),
|
||||
self._kube_upgrade_storage_stage(),
|
||||
]
|
||||
if self.is_simplex():
|
||||
stages.append(self._kube_host_cordon_stage())
|
||||
for ver in self.kube_versions:
|
||||
stages.append(self._kube_upgrade_first_control_plane_stage(
|
||||
ver))
|
||||
stages.extend(self._kube_upgrade_kubelet_stages(
|
||||
ver,
|
||||
self.std_controller_list,
|
||||
self.aio_controller_list,
|
||||
self.worker_list))
|
||||
if self.is_simplex():
|
||||
stages.append(self._kube_host_uncordon_stage())
|
||||
stages.extend([
|
||||
self._kube_upgrade_complete_stage(),
|
||||
self._kube_post_application_update_stage(),
|
||||
self._kube_upgrade_cleanup_stage(),
|
||||
])
|
||||
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
|
||||
|
||||
def test_resume_after_pre_app_update_failed(self):
|
||||
"""
|
||||
Test the kube_upgrade strategy creation when the upgrade had previously
|
||||
stopped with 'pre-updating-apps-failed'.
|
||||
It is expected to resume at the pre application update stage.
|
||||
"""
|
||||
kube_upgrade = self._create_kube_upgrade_obj(
|
||||
KUBE_UPGRADE_STATE.KUBE_PRE_UPDATING_APPS_FAILED,
|
||||
self.default_from_version,
|
||||
self.default_to_version)
|
||||
stages = [
|
||||
self._kube_pre_application_update_stage(),
|
||||
self._kube_upgrade_networking_stage(),
|
||||
self._kube_upgrade_storage_stage(),
|
||||
]
|
||||
if self.is_simplex():
|
||||
stages.append(self._kube_host_cordon_stage())
|
||||
for ver in self.kube_versions:
|
||||
stages.append(self._kube_upgrade_first_control_plane_stage(ver))
|
||||
stages.extend(self._kube_upgrade_kubelet_stages(
|
||||
ver,
|
||||
self.std_controller_list,
|
||||
self.aio_controller_list,
|
||||
self.worker_list))
|
||||
if self.is_simplex():
|
||||
stages.append(self._kube_host_uncordon_stage())
|
||||
stages.extend([
|
||||
self._kube_upgrade_complete_stage(),
|
||||
self._kube_post_application_update_stage(),
|
||||
self._kube_upgrade_cleanup_stage(),
|
||||
])
|
||||
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
|
||||
|
||||
def test_resume_after_pre_app_update_succeeded(self):
|
||||
"""
|
||||
Test the kube_upgrade strategy creation when the upgrade had previously
|
||||
stopped with 'pre-updated-apps'.
|
||||
It is expected to resume at the upgrade networking stage.
|
||||
"""
|
||||
kube_upgrade = self._create_kube_upgrade_obj(
|
||||
KUBE_UPGRADE_STATE.KUBE_PRE_UPDATED_APPS,
|
||||
self.default_from_version,
|
||||
self.default_to_version)
|
||||
stages = [
|
||||
self._kube_upgrade_networking_stage(),
|
||||
self._kube_upgrade_storage_stage(),
|
||||
@ -717,6 +781,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
|
||||
stages.append(self._kube_host_uncordon_stage())
|
||||
stages.extend([
|
||||
self._kube_upgrade_complete_stage(),
|
||||
self._kube_post_application_update_stage(),
|
||||
self._kube_upgrade_cleanup_stage(),
|
||||
])
|
||||
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
|
||||
@ -744,6 +809,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
|
||||
stages.append(self._kube_host_uncordon_stage())
|
||||
stages.extend([
|
||||
self._kube_upgrade_complete_stage(),
|
||||
self._kube_post_application_update_stage(),
|
||||
self._kube_upgrade_cleanup_stage(),
|
||||
])
|
||||
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
|
||||
@ -769,6 +835,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
|
||||
stages.append(self._kube_host_uncordon_stage())
|
||||
stages.extend([
|
||||
self._kube_upgrade_complete_stage(),
|
||||
self._kube_post_application_update_stage(),
|
||||
self._kube_upgrade_cleanup_stage(),
|
||||
])
|
||||
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
|
||||
@ -801,6 +868,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
|
||||
stages.append(self._kube_host_uncordon_stage())
|
||||
stages.extend([
|
||||
self._kube_upgrade_complete_stage(),
|
||||
self._kube_post_application_update_stage(),
|
||||
self._kube_upgrade_cleanup_stage(),
|
||||
])
|
||||
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
|
||||
@ -832,6 +900,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
|
||||
stages.append(self._kube_host_uncordon_stage())
|
||||
stages.extend([
|
||||
self._kube_upgrade_complete_stage(),
|
||||
self._kube_post_application_update_stage(),
|
||||
self._kube_upgrade_cleanup_stage(),
|
||||
])
|
||||
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
|
||||
@ -863,6 +932,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
|
||||
stages.append(self._kube_host_uncordon_stage())
|
||||
stages.extend([
|
||||
self._kube_upgrade_complete_stage(),
|
||||
self._kube_post_application_update_stage(),
|
||||
self._kube_upgrade_cleanup_stage(),
|
||||
])
|
||||
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
|
||||
@ -892,6 +962,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
|
||||
stages.append(self._kube_host_uncordon_stage())
|
||||
stages.extend([
|
||||
self._kube_upgrade_complete_stage(),
|
||||
self._kube_post_application_update_stage(),
|
||||
self._kube_upgrade_cleanup_stage(),
|
||||
])
|
||||
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
|
||||
@ -919,6 +990,7 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
|
||||
stages.append(self._kube_host_uncordon_stage())
|
||||
stages.extend([
|
||||
self._kube_upgrade_complete_stage(),
|
||||
self._kube_post_application_update_stage(),
|
||||
self._kube_upgrade_cleanup_stage(),
|
||||
])
|
||||
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
|
||||
@ -946,10 +1018,42 @@ class TestSimplexApplyStrategy(sw_update_testcase.SwUpdateStrategyTestCase,
|
||||
stages.append(self._kube_host_uncordon_stage())
|
||||
stages.extend([
|
||||
self._kube_upgrade_complete_stage(),
|
||||
self._kube_post_application_update_stage(),
|
||||
self._kube_upgrade_cleanup_stage(),
|
||||
])
|
||||
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
|
||||
|
||||
def test_resume_after_post_app_update_failed(self):
|
||||
"""
|
||||
Test the kube_upgrade strategy creation when the upgrade had previously
|
||||
stopped with 'post-updating-apps-failed'
|
||||
It is expected to resume at the post application update stage.
|
||||
"""
|
||||
kube_upgrade = self._create_kube_upgrade_obj(
|
||||
KUBE_UPGRADE_STATE.KUBE_POST_UPDATING_APPS_FAILED,
|
||||
self.default_from_version,
|
||||
self.default_to_version)
|
||||
stages = [
|
||||
self._kube_post_application_update_stage(),
|
||||
self._kube_upgrade_cleanup_stage(),
|
||||
]
|
||||
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
|
||||
|
||||
def test_resume_after_post_app_update_succeeded(self):
|
||||
"""
|
||||
Test the kube_upgrade strategy creation when the upgrade had previously
|
||||
stopped with 'post-updated-apps'.
|
||||
It is expected to resume at the cleanup stage.
|
||||
"""
|
||||
kube_upgrade = self._create_kube_upgrade_obj(
|
||||
KUBE_UPGRADE_STATE.KUBE_POST_UPDATED_APPS,
|
||||
self.default_from_version,
|
||||
self.default_to_version)
|
||||
stages = [
|
||||
self._kube_upgrade_cleanup_stage(),
|
||||
]
|
||||
self.validate_apply_phase(self.is_simplex(), kube_upgrade, stages)
|
||||
|
||||
|
||||
@mock.patch('nfv_vim.event_log._instance._event_issue',
|
||||
sw_update_testcase.fake_event_issue)
|
||||
|
@ -117,6 +117,8 @@ from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_host_cordon # no
|
||||
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_host_uncordon # noqa: F401
|
||||
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_host_upgrade_control_plane # noqa: F401
|
||||
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_host_upgrade_kubelet # noqa: F401
|
||||
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_post_application_update # noqa: F401
|
||||
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_pre_application_update # noqa: F401
|
||||
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_rootca_update_abort # noqa: F401
|
||||
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_rootca_update_complete # noqa: F401
|
||||
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_kube_rootca_update_generate_cert # noqa: F401
|
||||
|
@ -321,6 +321,26 @@ def nfvi_kube_upgrade_start(to_version, force, alarm_ignore_list, callback):
|
||||
return cmd_id
|
||||
|
||||
|
||||
def nfvi_kube_pre_application_update(callback):
|
||||
"""
|
||||
Kube Upgrade Pre Application Update
|
||||
"""
|
||||
cmd_id = _infrastructure_plugin.invoke_plugin(
|
||||
'kube_pre_application_update',
|
||||
callback=callback)
|
||||
return cmd_id
|
||||
|
||||
|
||||
def nfvi_kube_post_application_update(callback):
|
||||
"""
|
||||
Kube Upgrade Post Application Update
|
||||
"""
|
||||
cmd_id = _infrastructure_plugin.invoke_plugin(
|
||||
'kube_post_application_update',
|
||||
callback=callback)
|
||||
return cmd_id
|
||||
|
||||
|
||||
def nfvi_get_kube_host_upgrade_list(callback):
|
||||
"""
|
||||
Get kube host upgrade list
|
||||
|
@ -37,12 +37,16 @@ class KubeUpgradeState(Constants):
|
||||
Maintaining the same order as defined in kubernetes.py
|
||||
"""
|
||||
|
||||
KUBE_UPGRADE_STARTING = Constant('upgrade-starting')
|
||||
KUBE_UPGRADE_STARTING_FAILED = Constant('upgrade-starting-failed')
|
||||
KUBE_UPGRADE_STARTED = Constant('upgrade-started')
|
||||
KUBE_UPGRADE_DOWNLOADING_IMAGES = Constant('downloading-images')
|
||||
KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED = Constant('downloading-images-failed')
|
||||
KUBE_UPGRADE_DOWNLOADED_IMAGES = Constant('downloaded-images')
|
||||
KUBE_PRE_UPDATING_APPS = Constant('pre-updating-apps')
|
||||
KUBE_PRE_UPDATING_APPS_FAILED = Constant('pre-updating-apps-failed')
|
||||
KUBE_PRE_UPDATED_APPS = Constant('pre-updated-apps')
|
||||
KUBE_POST_UPDATING_APPS = Constant('post-updating-apps')
|
||||
KUBE_POST_UPDATING_APPS_FAILED = Constant('post-updating-apps-failed')
|
||||
KUBE_POST_UPDATED_APPS = Constant('post-updated-apps')
|
||||
KUBE_UPGRADING_NETWORKING = Constant('upgrading-networking')
|
||||
KUBE_UPGRADING_NETWORKING_FAILED = Constant('upgrading-networking-failed')
|
||||
KUBE_UPGRADED_NETWORKING = Constant('upgraded-networking')
|
||||
|
@ -21,6 +21,8 @@ from nfv_vim.strategy._strategy_steps import KubeHostCordonStep # noqa: F401
|
||||
from nfv_vim.strategy._strategy_steps import KubeHostUncordonStep # noqa: F401
|
||||
from nfv_vim.strategy._strategy_steps import KubeHostUpgradeControlPlaneStep # noqa: F401
|
||||
from nfv_vim.strategy._strategy_steps import KubeHostUpgradeKubeletStep # noqa: F401
|
||||
from nfv_vim.strategy._strategy_steps import KubePostApplicationUpdateStep # noqa: F401
|
||||
from nfv_vim.strategy._strategy_steps import KubePreApplicationUpdateStep # noqa: F401
|
||||
from nfv_vim.strategy._strategy_steps import KubeRootcaUpdateCompleteStep # noqa: F401
|
||||
from nfv_vim.strategy._strategy_steps import KubeRootcaUpdateGenerateCertStep # noqa: F401
|
||||
from nfv_vim.strategy._strategy_steps import KubeRootcaUpdateHostTrustBothcasStep # noqa: F401
|
||||
|
@ -3211,7 +3211,23 @@ class KubeUpgradeStrategy(SwUpdateStrategy,
|
||||
strategy.STRATEGY_STAGE_NAME.KUBE_UPGRADE_DOWNLOAD_IMAGES)
|
||||
stage.add_step(strategy.KubeUpgradeDownloadImagesStep())
|
||||
self.apply_phase.add_stage(stage)
|
||||
# Next stage after download images is upgrade networking
|
||||
# Next stage after download images is pre application update
|
||||
self._add_kube_pre_application_update_stage()
|
||||
|
||||
def _add_kube_pre_application_update_stage(self):
|
||||
"""
|
||||
Add kube pre application update stage.
|
||||
This stage only occurs after download images
|
||||
It then proceeds to the next stage
|
||||
"""
|
||||
from nfv_vim import strategy
|
||||
|
||||
stage = strategy.StrategyStage(
|
||||
strategy.STRATEGY_STAGE_NAME.KUBE_PRE_APPLICATION_UPDATE)
|
||||
stage.add_step(strategy.KubePreApplicationUpdateStep())
|
||||
self.apply_phase.add_stage(stage)
|
||||
|
||||
# Next stage after pre application update is upgrade networking
|
||||
self._add_kube_upgrade_networking_stage()
|
||||
|
||||
def _add_kube_upgrade_networking_stage(self):
|
||||
@ -3496,13 +3512,29 @@ class KubeUpgradeStrategy(SwUpdateStrategy,
|
||||
strategy.STRATEGY_STAGE_NAME.KUBE_UPGRADE_COMPLETE)
|
||||
stage.add_step(strategy.KubeUpgradeCompleteStep())
|
||||
self.apply_phase.add_stage(stage)
|
||||
# stage after kube upgrade complete stage, cleans up the kube upgrade
|
||||
# Next stage after upgrade complete is post application update
|
||||
self._add_kube_post_application_update_stage()
|
||||
|
||||
def _add_kube_post_application_update_stage(self):
|
||||
"""
|
||||
Add kube post application update stage.
|
||||
This stage occurs after the upgrade is completed
|
||||
It then proceeds to the next stage
|
||||
"""
|
||||
from nfv_vim import strategy
|
||||
|
||||
stage = strategy.StrategyStage(
|
||||
strategy.STRATEGY_STAGE_NAME.KUBE_POST_APPLICATION_UPDATE)
|
||||
stage.add_step(strategy.KubePostApplicationUpdateStep())
|
||||
self.apply_phase.add_stage(stage)
|
||||
|
||||
# Next stage after post application update is upgrade cleanup
|
||||
self._add_kube_upgrade_cleanup_stage()
|
||||
|
||||
def _add_kube_upgrade_cleanup_stage(self):
|
||||
"""
|
||||
kube upgrade cleanup stage deletes the kube upgrade.
|
||||
This stage occurs after all kube upgrade is completed
|
||||
This stage occurs after the post application update stage
|
||||
"""
|
||||
from nfv_vim import strategy
|
||||
stage = strategy.StrategyStage(
|
||||
@ -3563,10 +3595,6 @@ class KubeUpgradeStrategy(SwUpdateStrategy,
|
||||
# Note: there are no resume states for actions that are still running
|
||||
# ie: KUBE_UPGRADE_DOWNLOADING_IMAGES
|
||||
RESUME_STATE = {
|
||||
# If upgrade start failed, allow to restart
|
||||
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTING_FAILED:
|
||||
self._add_kube_upgrade_start_stage,
|
||||
|
||||
# after upgrade-started -> download images
|
||||
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTED:
|
||||
self._add_kube_upgrade_download_images_stage,
|
||||
@ -3575,8 +3603,16 @@ class KubeUpgradeStrategy(SwUpdateStrategy,
|
||||
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_DOWNLOADING_IMAGES_FAILED:
|
||||
self._add_kube_upgrade_download_images_stage,
|
||||
|
||||
# After downloading images -> upgrade networking
|
||||
# After downloading images -> pre update applications
|
||||
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_DOWNLOADED_IMAGES:
|
||||
self._add_kube_pre_application_update_stage,
|
||||
|
||||
# if pre updating apps failed, resume at pre updating apps
|
||||
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_PRE_UPDATING_APPS_FAILED:
|
||||
self._add_kube_pre_application_update_stage,
|
||||
|
||||
# After pre updating apps -> upgrade networking
|
||||
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_PRE_UPDATED_APPS:
|
||||
self._add_kube_upgrade_networking_stage,
|
||||
|
||||
# if networking state failed, resync at networking state
|
||||
@ -3631,8 +3667,16 @@ class KubeUpgradeStrategy(SwUpdateStrategy,
|
||||
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_HOST_UNCORDON_COMPLETE:
|
||||
self._add_kube_upgrade_complete_stage,
|
||||
|
||||
# upgrade is completed, delete the upgrade
|
||||
# upgrade is completed, post update apps
|
||||
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_COMPLETE:
|
||||
self._add_kube_post_application_update_stage,
|
||||
|
||||
# If post updating apps failed, resume at post application update stage
|
||||
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_POST_UPDATING_APPS_FAILED:
|
||||
self._add_kube_post_application_update_stage,
|
||||
|
||||
# After post updating apps, delete the upgrade
|
||||
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_POST_UPDATED_APPS:
|
||||
self._add_kube_upgrade_cleanup_stage,
|
||||
}
|
||||
|
||||
|
@ -63,6 +63,7 @@ class StrategyStageNames(Constants):
|
||||
KUBE_UPGRADE_QUERY = Constant('kube-upgrade-query')
|
||||
KUBE_UPGRADE_START = Constant('kube-upgrade-start')
|
||||
KUBE_UPGRADE_DOWNLOAD_IMAGES = Constant('kube-upgrade-download-images')
|
||||
KUBE_PRE_APPLICATION_UPDATE = Constant('kube-pre-application-update')
|
||||
KUBE_UPGRADE_FIRST_CONTROL_PLANE = \
|
||||
Constant('kube-upgrade-first-control-plane')
|
||||
KUBE_UPGRADE_NETWORKING = Constant('kube-upgrade-networking')
|
||||
@ -73,6 +74,7 @@ class StrategyStageNames(Constants):
|
||||
KUBE_UPGRADE_KUBELETS_CONTROLLERS = \
|
||||
Constant('kube-upgrade-kubelets-controllers')
|
||||
KUBE_UPGRADE_KUBELETS_WORKERS = Constant('kube-upgrade-kubelets-workers')
|
||||
KUBE_POST_APPLICATION_UPDATE = Constant('kube-post-application-update')
|
||||
KUBE_UPGRADE_COMPLETE = Constant('kube-upgrade-complete')
|
||||
KUBE_UPGRADE_CLEANUP = Constant('kube-upgrade-cleanup')
|
||||
|
||||
|
@ -88,6 +88,8 @@ class StrategyStepNames(Constants):
|
||||
KUBE_HOST_UPGRADE_CONTROL_PLANE = \
|
||||
Constant('kube-host-upgrade-control-plane')
|
||||
KUBE_HOST_UPGRADE_KUBELET = Constant('kube-host-upgrade-kubelet')
|
||||
KUBE_PRE_APPLICATION_UPDATE = Constant('kube-pre-application-update')
|
||||
KUBE_POST_APPLICATION_UPDATE = Constant('kube-post-application-update')
|
||||
# system config update specific steps
|
||||
QUERY_SYSTEM_CONFIG_UPDATE_HOSTS = Constant('query-system-config-update-hosts')
|
||||
SYSTEM_CONFIG_UPDATE_HOSTS = Constant('system-config-update-hosts')
|
||||
@ -4134,8 +4136,7 @@ class KubeUpgradeStartStep(AbstractKubeUpgradeStep):
|
||||
super(KubeUpgradeStartStep, self).__init__(
|
||||
STRATEGY_STEP_NAME.KUBE_UPGRADE_START,
|
||||
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTED,
|
||||
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_UPGRADE_STARTING_FAILED,
|
||||
timeout_in_secs=1800)
|
||||
None) # there is no failure state if upgrade-start fails
|
||||
# next 2 attributes must be persisted through from_dict/as_dict
|
||||
self._to_version = to_version
|
||||
self._force = force
|
||||
@ -4171,9 +4172,13 @@ class KubeUpgradeStartStep(AbstractKubeUpgradeStep):
|
||||
response = (yield)
|
||||
DLOG.debug("%s callback response=%s." % (self._name, response))
|
||||
|
||||
# kube-upgrade-start will return a result when it completes,
|
||||
# so we do not want to use handle_event
|
||||
if response['completed']:
|
||||
if self.strategy is not None:
|
||||
self.strategy.nfvi_kube_upgrade = response['result-data']
|
||||
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
|
||||
self.stage.step_complete(result, "")
|
||||
else:
|
||||
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
||||
self.stage.step_complete(result, response['reason'])
|
||||
@ -4363,6 +4368,82 @@ class KubeUpgradeDownloadImagesStep(AbstractKubeUpgradeStep):
|
||||
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
||||
|
||||
|
||||
class KubePreApplicationUpdateStep(AbstractKubeUpgradeStep):
|
||||
"""Kube Pre Application Update - Strategy Step"""
|
||||
|
||||
def __init__(self):
|
||||
from nfv_vim import nfvi
|
||||
super(KubePreApplicationUpdateStep, self).__init__(
|
||||
STRATEGY_STEP_NAME.KUBE_PRE_APPLICATION_UPDATE,
|
||||
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_PRE_UPDATED_APPS,
|
||||
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_PRE_UPDATING_APPS_FAILED,
|
||||
timeout_in_secs=1800)
|
||||
|
||||
def abort(self):
|
||||
"""
|
||||
Returns the abort step related to this step
|
||||
"""
|
||||
return [KubeUpgradeAbortStep()]
|
||||
|
||||
@coroutine
|
||||
def _response_callback(self):
|
||||
"""Kube Pre Application Update - Callback"""
|
||||
|
||||
response = (yield)
|
||||
DLOG.debug("%s callback response=%s." % (self._name, response))
|
||||
|
||||
if response['completed']:
|
||||
if self.strategy is not None:
|
||||
self.strategy.nfvi_kube_upgrade = response['result-data']
|
||||
else:
|
||||
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
||||
self.stage.step_complete(result, response['reason'])
|
||||
|
||||
def apply(self):
|
||||
"""Kube Pre Application Update """
|
||||
|
||||
from nfv_vim import nfvi
|
||||
DLOG.info("Step (%s) apply." % self._name)
|
||||
|
||||
nfvi.nfvi_kube_pre_application_update(self._response_callback())
|
||||
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
||||
|
||||
|
||||
class KubePostApplicationUpdateStep(AbstractKubeUpgradeStep):
|
||||
"""Kube Post Application Update - Strategy Step"""
|
||||
|
||||
def __init__(self):
|
||||
from nfv_vim import nfvi
|
||||
super(KubePostApplicationUpdateStep, self).__init__(
|
||||
STRATEGY_STEP_NAME.KUBE_POST_APPLICATION_UPDATE,
|
||||
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_POST_UPDATED_APPS,
|
||||
nfvi.objects.v1.KUBE_UPGRADE_STATE.KUBE_POST_UPDATING_APPS_FAILED,
|
||||
timeout_in_secs=1800)
|
||||
|
||||
@coroutine
|
||||
def _response_callback(self):
|
||||
"""Kube Post Application Update - Callback"""
|
||||
|
||||
response = (yield)
|
||||
DLOG.debug("%s callback response=%s." % (self._name, response))
|
||||
|
||||
if response['completed']:
|
||||
if self.strategy is not None:
|
||||
self.strategy.nfvi_kube_upgrade = response['result-data']
|
||||
else:
|
||||
result = strategy.STRATEGY_STEP_RESULT.FAILED
|
||||
self.stage.step_complete(result, response['reason'])
|
||||
|
||||
def apply(self):
|
||||
"""Kube Post Application Update """
|
||||
|
||||
from nfv_vim import nfvi
|
||||
DLOG.info("Step (%s) apply." % self._name)
|
||||
|
||||
nfvi.nfvi_kube_post_application_update(self._response_callback())
|
||||
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
|
||||
|
||||
|
||||
class KubeUpgradeNetworkingStep(AbstractKubeUpgradeStep):
|
||||
"""Kube Upgrade Networking - Strategy Step"""
|
||||
|
||||
@ -4894,6 +4975,8 @@ def strategy_step_rebuild_from_dict(data):
|
||||
STRATEGY_STEP_NAME.KUBE_UPGRADE_NETWORKING: KubeUpgradeNetworkingStep,
|
||||
STRATEGY_STEP_NAME.KUBE_UPGRADE_STORAGE: KubeUpgradeStorageStep,
|
||||
STRATEGY_STEP_NAME.KUBE_UPGRADE_START: KubeUpgradeStartStep,
|
||||
STRATEGY_STEP_NAME.KUBE_PRE_APPLICATION_UPDATE: KubePreApplicationUpdateStep,
|
||||
STRATEGY_STEP_NAME.KUBE_POST_APPLICATION_UPDATE: KubePostApplicationUpdateStep,
|
||||
STRATEGY_STEP_NAME.QUERY_KUBE_HOST_UPGRADE: QueryKubeHostUpgradeStep,
|
||||
STRATEGY_STEP_NAME.QUERY_KUBE_UPGRADE: QueryKubeUpgradeStep,
|
||||
STRATEGY_STEP_NAME.QUERY_KUBE_VERSIONS: QueryKubeVersionsStep,
|
||||
|
Loading…
Reference in New Issue
Block a user